{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 10 – Introduction to Artificial Neural Networks**\n",
    "\n",
    "**第10章 - 人工神经网络（ANN）简介**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercises in chapter 10._\n",
    "\n",
    "_这个notebook包含第10章所有的示例代码及练习题解答_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "# 让输出稳定\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph() # \n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perceptrons\n",
    "# 感知器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "# 二元分类，预测是否为Setosa\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # petal length, petal width 花瓣长度，宽度\n",
    "y = (iris.target == 0).astype(np.int) # 布尔转整数\n",
    "\n",
    "per_clf = Perceptron(max_iter=100, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure perceptron_iris_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmcTvX7x/HXNYshjH2pbCEpyV5KRZtSlNL6tWSJkJZfixSlMvYZRJZUVGQtipSlSHYz8i1fLZKisi/ZtzGf3x/3uBtjNmPmPrO8n4/HeXTf1/mcc657huly5nM+lznnEBERERERnyCvExARERERyUpUIIuIiIiIJKACWUREREQkARXIIiIiIiIJqEAWEREREUlABbKIiIiISAIqkEVEREREEghYgWxmYWb2npltNrODZrbWzJqkMP7/zGy7me03s3FmFpZgXwUzW2RmR8zsZzO7NTCfQkRERERyukDeQQ4B/gQaAoWAV4BpZlYh8UAzux3oAdwCVAAqAq8nGDIZWAsUA3oCH5tZicxLXURERERyC/Oyk56Z/QC87pz7JFF8EvCHc+7l+Pe3AB8550qbWRVgHVDcOXcwfv+S+P1jAvsJRERERCSnCfHqwmZWCqgCrE9idzXgswTvvwdKmVmx+H2bThfHCfZXS+Y6nYBOAPnz561z2WUXZ0D2me/48ZP8/vsOjhw57o8FB4dSrNgl5M1b0MPMRERERLyxZUvy+8qVS8sxf+DcbkvtOp4UyGYWCnwEfOCc+zmJIQWA/Qnen35dMIl9p/cnWfk658YCYwHq1KnsVq6MOo/MA+vEiZO8+upHDBnyKQCnTp1k165fufPOV7jzzlcIDvbs3zciIiIiAde5c/L7Xn45LcfUTdN1Ar6KhZkFAROAE0C3ZIYdAsITvD/9+mAS+07vP0gOkydPKAMGtGXWrFcoXtz3kZ1zzJnzBkOH3szevX96nKGIiIhIzhPQAtnMDHgPKAW0cM6dTGboeqBGgvc1gB3OuT3x+yqaWcFE+5OaqpEj3HFHHWJihtGoUXV/bOPGJfTtW5Pvv5/lYWYiIiIigROe+BZpKvHU9iUnoA/pmdkYoCZwq3PuUArj7gDeB24GtgGfAKudcz3i968ElgK9gCbAeOBS59yulK6f3aZYJHbq1CkGDvyEN96YQlxcnD9+001Pcd99gwgNDUvhaBEREZHcrXNnW+OcS3WeRSDXQS4PPI6vQN5uZofit5ZmVi7+dTkA59xcYBCwCNgcv/VOcLqH8U0i2QcMAO5PrTjOCYKDg3n55Qf56qs+lClTzB9ftGg4gwZdy44dGzzMTkRERCRn8HSZt0DL7neQE9q79yAdO45g9uzV/lhYWH4eeWQ09eu39jAzERERkawpy91BloxVtGhBPv74JYYN60iePL7VLI4fP8z777fh/fcf5dixZGewiIiIiEgKVCBnY2ZG1653sXTpIC699CJ/fOXKD+nfvw5//vlfD7MTERERyZ40xSKHOHToKE89NZaJExf5YyEheWjRIopGjZ7At4CIiIiISO7SvTscOHD6XV2ci0m1KNId5ByiQIF8jBv3NO+99zT58+cFIDb2BFOnPsmYMfdy+PBejzMUERERCbx/i+O0U4Gcw7RufROrVkVRo8Yl/tj3339GRERNNm5c6mFmIiIiItmDCuQcqEqVi1m6dBDdujX1x/bt+5OoqIZ88UUEcXGnPMxOREREJGtTgZxDhYWFMmTIY3zyycsULeprOuhcHLNmvcKbb97GP/9s9ThDERERkaxJBXIO16zZ1URHD6FBg8v9sV9+WURERA3+978vPcxMREREJGtSgZwLlC1bggULIujZ8yH/ahaHDu3mrbfu5OOPnyc29oTHGYqIiIhkjvDwcz9Gy7zlMt98s45HHx3Ctm37/LHy5evx2GNTKFGiooeZiYiIiGQuddKTJDVqVJ2YmGE0aVLHH9u8OZq+fWsREzPVw8xEREREsgYVyLlQiRKFmDmzJ4MGtSM01Nem+tixA7z77sNMmNCREyeOeJyhiIiIiHdUIOdSQUFBPPPMPSxe3J+KFUv548uWvUv//vX4++91HmYnIiIi4h3NQRYOHDhC166jmTZtiT8WGpqXBx4Yxg03dFKbahEREclwnTsnv2/MmKTjXbpAUqWrGYwenZZrag6ypFF4+AVMmPAsY8d2I1++PACcPHmMSZM68847D3HkyD8eZygiIiKSdHGcUjy9VCALAGZG27a3smJFFFdeWd4f/+676fTtW4tNm1Z6mJ2IiIhI4KhAljNccUVZli0bxOOP3+GP7dnzB5GRNzBv3iDi4uI8zE5EREQk86lAlrPkyxfGiBGdmTy5O4UKXQBAXFwsM2e+yFtvNeHAgR0eZygiIiKSeQJaIJtZNzOLMbPjZvZ+CuPGmNmhBNtxMzuYYP83ZnYswf5fAvIBcpkWLa4jOnoo11xzmT/244/ziYiowU8/feVhZiIiIiKZJ9B3kLcCEcC4lAY55zo75wqc3oDJwPREw7olGHNZEqeRDFChQikWLuzLCy+08McOHNjB8OGN+fTTlzl16qSH2YmIiEhuktzCWhm94JYny7yZWQRQxjnXNg1j8wPbgabOucXxsW+Aic65d8/lulrm7fwsWLCWdu2GsXPnfn+sYsVr6dBhMsWKlU/hSBERERHv5aRl3loAu4BvE8X7m9luM1tmZo2SO9jMOsVP64jZvftAZuaZ4912Wy1iYoZx6601/LFNm1bQt29N1q6d4WFmIiIiIhknOxTIjwIfujNvdb8IVAQuBsYCs82sUlIHO+fGOufqOufqFi8envnZ5nClSxfh8897ExHRmuBg3x+fI0f+4e23WzBpUldOnDjqcYYiIiIi5ydLF8hmVhZoCHyYMO6cW+WcO+icO+6c+wBYBtzpRY65UVBQEN27t2DRon6UL1/CH//229EMHFifbdt+8jA7ERERkfMT4nUCqWgDLHfObUplnAPUDznA6tevSnT0UB5/fCQzZ64A4O+/f6B//7o8/PBbXHttW7WpFhER8Vj37nAgiVmm4eEwaFDg8wm0Mz9/nTppOSbQy7yFmFleIBgINrO8ZpZSkd4GeD/ROQqb2e2njzWzlsCNwLxMS1ySVbhwAaZM6c5bb3UmLCwUgBMnjvDhh+0ZN64VR49q3reIiIiXkiqOU4rnNOn5nIGeYtELOAr0AFrFv+5lZuXi1zMud3qgmV0LlOHs5d1C8S0VtwvYDTwJNHfOaS1kj5gZnTrdwfLlg6latYw/Hh09iX79arN5c4yH2YmIiIicm4AWyM6515xzlmh7zTm3JX494y0Jxq5wzuV3zh1MdI5dzrl6zrmCzrnCzrn6zrkFgfwckrTq1SuwYkUk7drd6o/t2vUbkZHX8tVXQ/FiSUERERGRc5WlH9KT7Cd//ry8/XY3PvzwWQoWzAfAyZOxfPzxs4wa1YxDh3Z7nKGIiIhIylQgS6Z4+OEbWb16CHXqVPbH1q2bQ58+NdiwYbGHmYmIiIikTAWyZJpKlS5k8eL+/N//3eOP7d+/laFDb2b27Nc4dSrWw+xERERyh/Bk2kAkF89p0vM5PWk17RW1mvbOl1/G0KHDcBJ2M7z00htp3/4jihQpk8KRIiIiIhkjJ7WalhygSZO6REcPpWHDK/2xX3/9loiIGvzww2wPMxMRERE5kwpkCZiLLy7G3Lmv07v3IwQF+f7oHT68l1Gj7mbatGc4efK4xxmKiIiIqECWAAsODqZnz4f46qs+lClTzB9fuPBNBg++jh07fvUwOxERERHNQRYP7dlzgI4d3+Lzz1f7Y2FhBfjPf0ZzzTWtPMxMREQk5whEq+ns0s5ac5AlyytWLJxPPnmJoUMfI08eX8fx48cPMX58a95/vy3Hjh3yOEMREZHsLxCtpnNaO2sVyOIpM+OJJ5qyZMlAKle+yB9fufID+vevy59//tfD7ERERCQ3UoEsWUKtWpVYtSqKli0b+WM7dvzCwIH1+eabkWpTLSIiIgGjAlmyjIIF8zF+/DO8997T5M+fF4DY2ONMmdKNMWPu4/DhvR5nKCIiIrmBCmTJclq3vomVK6OoUeMSf+z77z8lIqImGzcu8zAzERERyQ1UIEuWdNllF7NkyUCeeOIuf2zfvj8ZMqQhX3zRl7i4Ux5mJyIikn0EotV0TmtnrWXeJMubNWsVHTuOYN++f1e1uOyym2nffiKFCl3oYWYiIiKSnWiZN8kx7r77GqKjh9KgweX+2C+/LCQiogbr18/1MDMRERHJiVQgS7ZQrlwJFiyI4KWXHsDMADh4cBcjRjThk0+6Ext7wuMMRUREJKdQgSzZRkhIMK+/3pK5c1/nwguL+OMLFgwmMvIGdu3a5GF2IiIiklMEdA6ymXUD2gLVgcnOubbJjGsLvAccTRBu6pz7Jn5/BWA8cA2wBejmnPsqtetrDnLOsXPnPzz22HDmzv3OH8ubN5xWrd6hbt0HPcxMRERyo6zcarlz5+T3jRlzdiw9nyVQn79LF0iqdDWD0aPTkltdnIux1K4T6DvIW4EIYFwaxq5wzhVIsH2TYN9kYC1QDOgJfGxmJTI8W8mySpYszKef9mLgwLaEhAQDcOzYAd599yEmTuzEiRNHPM5QRERyk5zUajk9nyVQnz+5+7op3e9NTw4BLZCdczOcc58Ce9J7DjOrAtQGejvnjjrnPgHWAS0yKE3JJoKCgvi//2vO4sX9qVixlD++dOk79O9fj7///p+H2YmIiEh2lZXnINcys91mtsHMXjGzkPh4NWCTc+5ggrHfx8fPYmadzCzGzGJ2786G/4yTVNWrV4VVq4bwwAPX+2Pbtv3IgAH1WLLkHbWpFhERkXOSVQvkb4ErgZL47gw/ArwQv68AsD/R+P1AwaRO5Jwb65yr65yrW7x4Nl2tWlJVqFB+Jk58jrfffoJ8+fIAcPLkMT76qBPvvvswR48m/iMjIiIikrQsWSA75zY55353zsU559YBbwD3x+8+BCSudMOBg0iuZma0a3cbK1ZEUa1aOX98zZppRETU5PffV3mYnYiIiGQXWbJAToIDTj9xuB6oaGYJ7xjXiI+LcMUVZVm+fDAdO97uj+3Z8wdRUQ2YP38wcXFxHmYnIiI5UU5qtZyezxKoz2/JrD+RXDy9OQR6mbcQIAToDZQBOgKxzrnYROOaAN8553aYWVXgY2C6c+71+P0rgaVAL6AJviXfLnXO7Urp+lrmLff5+ONldOkykv37/13V4oorbqdt2w8JDy/pYWYiIiISaFm11XQvfGsb9wBaxb/uZWblzOyQmZ3+vfgtwA9mdhj4ApgB9EtwnoeBusA+YABwf2rFseRO99/fgNWrh3L11VX8sR9/nEdERA1+/vlrDzMTERGRrCqgd5C9pjvIudfJk7H07j2JyMgZ/piZcfvtL9Gs2esEB4ekcLSIiIjkBFn1DrKIJ0JDQ+jXrw1z5vSmZMlCADjnmDu3H1FRDdm7d4vHGYqIiEhWoTvIkuts376Pdu2G8fXX3/tjhQvn56GHJlCr1r0eZiYiIinJyu2cAyU9rZbPVXq+zuffAjpt1zlfuoMskozSpYswZ05v+vRpRXCw76/AP/8c5u2372Py5Cc4efKYxxmKiEhSclI75/RKT6vlc5Wer3NGtoDOCt9PFciSKwUFBfHii/ezcGE/ypUr4Y8vXjyKAQOuYfv2nz3MTkRERLykAllytWuvrUp09FCaN6/vj/399w/061eH5cvfV5tqERGRXEgFsuR6RYoUYOrUFxkx4nHCwkIBOHHiCB9+2I7x41tz7JiaNIqIiOQmKpBF8C359vjjTVi2bDCXXVbGH1+9+iP69q3N5s1rPMxOREREAkkFskgCV11VgZUrI2nb9hZ/bNeujQwadC1ffz1MUy5ERDyUk9o5p1d6Wi2fq/R8nTOyBXRW+H5qmTeRZEyevJgnnhjNoUP/rmpRvXpTHn10PAUKFPcwMxEREUkPLfMmcp4eeaQhq1cPpXbtSv7YunWfExFRkw0bFnuYmYiIiGQmFcgiKahc+UK+/XYATz99tz/2zz9/M3TozXz++evExZ3yMDsRERHJDCqQRVKRJ08ogwe3Z+bMnhQrVhAA5+L4/PPXGDr0Fvbt+8vjDEVERCQjpblANrMLzOw6M2tuZvcl3DIzQZGs4q676hETM4wbb6zmj/3662IiImryww+fe5iZiIiIZKSQtAwys1uByUCxJHY7IDgjkxLJqi6+uBjz5r1B//4fExExlbi4OA4f3sOoUc24+eZnuPfeAYSGhnmdpojIeenePel2v+HhMGhQ4PM5H507J79vzJik4126JN0i2QxGj/b2mPR8b871mJz0/U+vtN5BfhOYA5RxzgUl2lQcS64SHBxMr14PMX/+G1x88b//Zly4cBiDBzdg586NHmYnInL+kiqOUornNMkt8JXSwl+BOiY935tzPSa3f/8h7QVyBaCPc25rJuYikq3ceOOVREcP5c47/10tZsuWNfTtW4vVqyd5mJmIiIicj7QWyMuAyzIzEZHsqHjxcGbO7ElUVAdCQ30zlo4fP8S4cS358MP2HD9+2OMMRURE5FwlOwfZzGoneDsGiDSzi4B1wMmEY51z32VOeiJZn5nx5JPNaNDgclq1imLjxm0ALF8+nt9+W07HjlMpU6aGx1mKiIhIWqV0BzkGiI7/78dAVWAssCI+FpNgTJqYWTczizGz42b2fgrjHjWzNWZ2wMz+MrNBZhaSYP83ZnbMzA7Fb7+kNQeRzFK7dmVWrRrCI4809Md27PiFAQOu4ZtvRqlNtYiISDaRUoF8CVAx/r8pbRXP4XpbgQhgXCrjLgCeAYoD1wC3AM8nGtPNOVcgftP0D8kSChbMx/vvP8O77z7FBRf4VrOIjT3OlClP8PbbLTh8eJ/HGYqIpC48/NziOY3ZucUDeUx6vjfnekxu//4DWFruapnZjcBy51xsongIcJ1z7ttzuqhZBL4VMdqmcfyzwE3OuWbx778BJjrn3j2X69apU9mtXBl1LoeIpNvPP/9Fq1aR/PDDH/5Y0aLl6NBhMpUqXeddYiIiIrlU5862xjlXN7VxaX1IbxFQNIl4ofh9me1GYH2iWH8z221my8ysUXIHmlmn+GkdMbt356L1ScRzVauWYenSQXTpcqc/tnfvFqKibuTLL/upTbWIiEgWldYC2fA1BEmsGJCpj+mbWTugLhCZIPwivqkdF+ObFz3bzColdbxzbqxzrq5zrm7x4rnodwOSJeTNm4c33+zEtGk9KFKkAABxcaf47LOeDB9+O/v3b/c4QxEREUksxQLZzGaZ2Sx8xfHE0+/jtznAAmB5ZiVnZs2BAUAT59zu03Hn3Crn3EHn3HHn3Af4lqG7M7nziHitefP6REcP5brrLvfHfv75ayIiarB+/TwPMxMREZHEUms1vSf+vwbsA44m2HcCWAq8kwl5YWZ3xJ/7LufculSGu/gcRbKscuVK8NVXEbzxxhQGDvwY5xwHD+5kxIg7aNy4O/fcE0FwcKjXaYqIZFmBaLMcyNyy8nXOVVbNK71SLJCdc+0AzOwPINI5d17TKeIf6gsBgoFgM8sLxCbx8N/NwEfAvc651Yn2Fca3ssViIBZ4CN8c5WfOJzeRQAgJCeaNN1rSqNGVtG07jO3bfatazJ8/iF9/XUyHDpMpXvwSj7MUEcmaAtFmOb1y2nXOVVbNK73SNAfZOff6+RbH8XrhuwvdA2gV/7qXmZWLX8+4XPy4V/A9APhFgrWOv4zfF4pvqbhdwG7gSaC5c05rIUu2cfPNNYiJGUrjxrX8sd9/X0VERE3WrJnuYWYiIiKSUie930n6wbyzOOfStBayc+414LVkdhdIMO6mFM6xC6iXluuJZGUlSxZm1qxXGDZsFr16TSA29hTHjh3gnXce5OefO/HAA0PJk+cCr9MUERHJdVK6g/wWMDJ++wDfihW/ARPjt9/iY+9nbooiOVdQUBDPPtucxYv7c8klpfzxJUvGMmDA1Wzdmnh1QxEREclsyRbIzrmo0xu+jnkDnXO3Oedejd9uw7fCRJVAJSuSU9WrV4XVq4fQosW/DUS2bl1P//71WLr0XbWpFhERCaC0roN8HzAtifh04O6MS0ck9ypUKD+TJr3A6NFdyZs3DwAnTx5l4sSOvPfeIxw9ut/jDEVEvBWINsvpldOuc66yal7pldZW09uAVxK3djazx4AI51zpTMovQ6nVtGQX69dvoWXLSH78cYs/Vrz4JXToMIVLLrnaw8xERESyr4xuNT0UGGlmY8ysbfw2BhgRv09EMlC1auVYvnwwjz3W2B/bvft3oqKuY/78SOLi4jzMTkREJGdL6zJvg4DWQHVgSPxWHXjUOTcw89ITyb0uuCCMUaO68tFHzxMe7lvNIjb2FDNmvMDIkXdx4MBOjzMUERHJmdJ6Bxnn3DTnXAPnXNH4rYFzLql5ySKSgR544Hqio4dSr96l/tj69XPp27cmP/+80MPMREREcqbUWk2LSBZwySWlWLSoH6+++hFDhnwKwP7923jzzVu5446Xadr0NYKD9ddZJCk5rQVuTpGV20aLJHsH2cwOmFnx+NcH498nuQUuXZHcK0+eUAYMaMvs2a9SokQhAJxzfPllX4YMacTevVtSOYNI7pTTWuDmFFm5bbRISrecngQOJnithVhFsoDbb69NTMxQ2rUbxsKFPwDw22/LiIioSZs246hZs7nHGYqIiGRvyRbIzrkPErx+PyDZiEiaXHhhUebM6c3gwTN4/fXJnDoVx5Ej+xgz5l4aNepGixaDCQ3N63WaIiIi2VKaHtIzs5fMrL6ZBWd2QiKSNsHBwfTo8QBff92XsmWL++PffPMWAwfWZ/v2XzzMTkREJPtK6yoWdwGLgX/MbF58wXytCmYR71133eVERw/lnnvq+2N//fU9/fvXYcWKD1I4UkRERJKS1nWQrwcK42s5HY2vYF6Er2Cem3npiUhaFC1akGnTXmT48E6EhYUCcPz4YT74oC3jx7fm2LGDqZxBJOfKaS1wc4qs3DZaJE2tps84wKw0cBO+Ivkh4KRz7oJMyC3DqdW05Abff/87LVtGsmHD3/5YiRKV6dhxKuXK1fYwMxEREW9laKtpM3vAzEaZ2U/Ab0AnYCNwG1DkvDIVkQxVo8YlrFoVRZs2N/tju3ZtZNCga1m4cDjn+o9iERGR3Catc5CnAi2A8UAJ59xNzrnXnHPfOOeOZ156IpIe+fPn5d13n2L8+GcoUMC3mkVs7AmmTXua0aPv4dChPR5nKCIiknWltUB+HFiAbz3krWY228yeM7PaZmaZl56InI+WLRuxatUQatWq6I/98MNsIiJq8Ouv33qYmYiISNaVnjnIlYFG+KZX3Asccs4VTeOx3YC2QHVgsnOubQpj/w94EcgHfAJ0OX232swq4LubfQ2wBejmnPsqtetrDrLkVsePn6Rnzw8ZPny2P2YWRNOmvWnSpCdBQVqQRiTQunSBpP4XbAajR2e/62TVNtBqaS0JZegcZAAzCzKza/BNtXgA30N6AOey2OpWIAIYl8q1bgd6ALcAFYCKwOsJhkwG1gLFgJ7Ax2ZW4hzyEMlVwsJCiYzswIwZL1OsWEEAnItj9uzeDB16C/v2/Z3KGUQkoyV3fyqjHxMI1HWyahtotbSW9EjrQ3pfAPuAJfjuGq8F7geKOOeuTevFnHMznHOfAqlNgHwUeM85t945tw/og+/OM2ZWBagN9HbOHXXOfQKsw1e4i0gKmja9mujoodxwQzV/7NdfFxMRUYN16+Z4mJmIiEjWkdY7yD/gW9KtiHOuvnOuh3NurnPucCblVQ34PsH774FSZlYsft8m59zBRPurkQQz62RmMWYWs3u3/uknUqZMcebPf4NevR4iKMj3I+Dw4T2MHNmU6dOfJTb2hMcZioiIeCutjUIyuyBOrACwP8H7068LJrHv9P6CSZ3IOTfWOVfXOVe3eHGtJC4CvjbVr776CPPmvc5FF/37CMHXXw9l0KDr2Llzo4fZiYiIeCvNc5AD7BCQsJo9/fpgEvtO71erMJFz1LBhdWJihnHnnf8+r7Blyxr69atNdPRkDzMTERHxTlYtkNcDNRK8rwHscM7tid9X0cwKJtq/PoD5ieQYxYuHM3NmTyIj2xMaGgLAsWMHee+9//Dhhx04fjxQvzgSyV2SWyQ1oxdPDdR1smobaLW0lvQ452XezutiZiFACNAbKAN0BGKdc7GJxt0BvA/cDGzDt8zbaudcj/j9K4GlQC+gCb4l3y51zu1K6fpa5k0kZd99t5GWLSP57bft/ljp0lV57LGplClzlYeZiYiInL8MX+Ytg/QCjuJbwq1V/OteZlbOzA6ZWTkA59xcYBCwCNgcv/VOcJ6Hgbr4VtYYANyfWnEsIqmrXbsyq1YN4eGHb/THtm//mQEDrmbx4tFqUy0iIrlCQO8ge013kEXSxjnHhAkLeeqpsRw58m83+Vq1WtCq1Tvkz1/Ew+xERETS57zvIJvZQTM7kJYtY1MXEa+ZGW3a3MLKlVFUr17BH1+79hP69q3Fpk0rvEtOREQkk4WksK9bwLIQkSypatUyLF06kBdffJ8xY74EYO/ezURG3sDdd/ehceMX/Wspi4iI5BSaYiEiaTJz5goef/wt/vnn31Utqla9lXbtJlCoUGkPMxMREUmbrPqQnohkU/feey3R0UOpX/8yf+znn78iIqIGP/4438PMREREMlaaCmQzy2Nmr5vZBjM7ZmanEm6ZnaSIZA3ly5fk66/78uKL92Pxi6gePLiT4cNvZ+bMHpw6ddLjDEVERM5fWu8g9wEeBaKAOOAFYCSwB+iaOamJSFYUGhpCnz6t+OKL1yhVqrA/Pm/eQCIjb2T37t89zE5EROT8pbVAfhDo7Jx7GzgFfOacewrf2sS3ZVZyIpJ13XJLDWJihtG4cS1/7PffV9K3by3WrPnYw8xERETOT1oL5FLAj/GvDwGnbxvNBRpndFIikj2UKlWYWbNeoX//RwkJCQbg6NH9vPPOA3z0UWdOnDjqcYYiIiLnLq0F8hbgovjXG4Hb419fi68bnojkUkFBQTz33L0sWtSPChVK+uNLlrzNgAFXs3XrjykcLSIikvUdLgp1AAAgAElEQVSktUCeCdwS//pN4HUz+x14H3g3E/ISkWzmmmsuY/XqIbRocZ0/tnXr/+jfvy7Llr2nNtUiIpJtpKlAds695JzrG//6Y+B6YARwn3OuZybmJyLZSOHCBZg06QVGjepC3rx5ADh58igTJjzGe+/9h6NH93ucoYiISOrSuszbjWbm77rnnFvlnBsCzDWzGzMtOxHJdsyMxx67neXLB3P55WX98ZiYKfTtW5s//oj2MDsREZHUpXWKxSKgaBLxQvH7RETOcOWV5VmxIpIOHf5d6Gb37k0MGnQdCxZEERcX52F2IiIiyUtrgWxAUhMIiwGHk4iLiHDBBWGMHv0EEyc+T3j4BQDExcXyySfPM3JkUw4e3OVxhiIiImdLsUA2s1lmNgtfcTzx9Pv4bQ6wAFgeiERFJPt68MHrWb16CHXrXuqPrV//JRERNfjlF/0SSkREspbU7iDvid8M2Jfg/R7gL2AM0CozExSRnKFixdJ8800/nn22uT+2f/82hg27hVmzXuXUqVgPsxMREflXSEo7nXPtAMzsDyDSOafpFCKSbnnyhDJgQFsaNapOhw7D2bVrP845vviiDxs2LKJ9+0kULVo29ROJiIhkorQu8/a6c+6wmdU1s4fMLD+AmeVPuLqFiEha3HFHHaKjh3LTTdX9sY0blxIRUYP//vczDzMTERFJ+zJvpcxsFbAamISv9TTAECAqrRczs6JmNtPMDpvZZjP7TzLjvjSzQwm2E2a2LsH+P8zsaIL989Oag4hkDRddVJQvvniN119vSVCQ70fRkSP7GDOmOVOnPsXJk8c8zlBERHKrtK5iMRTYjm/ViiMJ4tOBxudwvZHACXwFdktgtJlVSzzIOdfEOVfg9IbvQcDpiYY1SzDmXHIQkSwiODiYl156gK+/jqBs2eL++KJFIxg06Fp27NjgYXYiIpJbpbVAvgXo6Zzblyj+G1AuLSeIn5bRAnjFOXfIObcUmAW0TuW4CsANwIQ05ioi2UyDBlcQHT2Uu+++xh/788//0q9fbVau/NDDzEREJDdKa4GcD9+d38RKAGn9PWgV4JRzLuEtoe+Bs+4gJ9IGWOKc+z1R/CMz22Vm882sRnIHm1knM4sxs5jduw+kMVURCbSiRQsyfXoP3nyzE3ny+B5tOH78MO+//yjjx7fh2LFDHmcoIiK5RVoL5G+BtgneOzMLBl4Evk7jOQoA+xPF9gMFUzmuDfB+olhLoAJQHl8nv3lmVjipg51zY51zdZ1zdYsXD09jqiLiBTOjS5c7Wbp0EJdeepE/vmrVBPr1q82WLWs9zE5ERHKLtBbI3YGOZrYACMP3YN6PQAPgpTSe4xCQuEINBw4md4CZXQ+UBj5OGHfOLXPOHXXOHXHO9Qf+wTcNQ0RygJo1K7JqVRStW9/kj+3c+SuDBtVn4cLhOJdUY08REZGMkdZl3n4ErgJWAPOBvPgemqvlnPstjdfaAISY2aUJYjWA9Skc8ygwwzmX2u9WHb5mJiKSQxQokI/33nuaceOeJn/+vADExp5g2rSnGT26OYcO7fE4QxERyanSegcZ59w259yrzrmmzrk7nXO9nHPbzuH4w8AM4I349ZMbAPeQzMN3ZpYPeIBE0yvMrJyZNTCzPGaW18xeAIoDy9Kai4hkH61a3cSqVVHUrFnRH/vhh1n07VuTX39d4mFmIiKSU6VYIJvZBWY20sz+NrOdZjbJzIqndEwquuJ74G8nMBno4pxbb2Y3mFniu8TN8c1RXpQoXhAYja/19d/AHUAT55xuJ4nkUFWqXMySJQN58smm/ti+fX8xZEgj5szpQ1zcKQ+zExGRnMZSmstnZoPxFbUf4Vut4hHgG+fcA4FJL2PVqVPZrVyZ5r4mIpIFzZ69mo4dR7B377+PL1Sp0oj27T+icOGLUjhSRERyu86dbY1zrm5q41KbYnEf0ME518k59xRwF9A8fgULEZGAa9bsamJihnL99Vf4Yxs2fENERA3WrfvCw8xERCSnSK1ALgv4J/k551YDsYBu04iIZ8qUKc78+X3o2fMhzHzP5x46tJuRI+/i44+fJzY2qWXbRURE0ia1AjmYsxuExAIhmZOOiEjahIQE07v3I8yf/wYXXVTUH//qqygGD27Arl1pXWBHRETkTKkVyAZMNLNZpzd8S7y9kygmIuKJhg2rEx09lCZN6vhjmzfH0LdvLaKjp3iYmYiIZFepFcgfAFuBPQm2icCfiWIiIp4pUaIQM2f2ZPDg9oSG+n7BdezYQd577xEmTHiM48cPe5yhiIhkJymuYpHTaBULkZxvzZqNtGoVyW+/bffHSpe+nI4dp3LxxdU9zExERLyWUatYiIhkK3XqVGbVqiE89NC/3ee3b/+JAQOu5ttvx6hNtYiIpEoFsogkaefOxcTEdGTZsnuJienIzp2LvU4pzcLDL+DDD59l7Nhu5MuXB4CTJ48xaVIXxo59gCNH/vE4QxERycpUIIvIWXbuXMxvv43i+PFdgOP48V389tuobFUkmxlt297KypVRXHlleX987dpPiIioyaZNKz3MTkREsjIVyCJyli1bJhIXd/yMWFzccbZsmehRRul3+eVlWbZsEI8/foc/tnfvZqKiGjBv3kDi4uI8zE5ERLIiFcgicpbjx3efUzyry5cvjBEjOjNlSncKFboAgFOn4pg5swcjRtzBgQM7PM5QRESyEhXIInKWsLDi5xTPLu677zqio4dSv/5l/thPPy0gIqIGP/64wMPMREQkK1GBLCJnKVeuFUFBYWfEgoLCKFeulUcZZZwKFUrx9dd96d69hb9N9YEDOxgx4nZmznyJU6dOepyhiIh4TQWyiJylZMmGVKrUlbCwEoARFlaCSpW6UrJkQ69TyxChoSFERLRmzpzelCxZCADnHPPmDSAy8kZ27/7D2wRFRMRTahQiIrna9u37aN9+GF999b0/li9fIVq3fo/atVt4mJmIiGQ0NQoREUmD0qWL8Pnnvenbtw3Bwb4fiUeP7mfs2PuZNKkLJ04c9ThDEREJNBXIIpLrBQUF8cIL97FoUT/Kly/hj3/77RgGDryGbdt+8jA7EREJNBXIIiLx6tevSnT0UO6991p/7O+/19G/f12WLRunNtUiIrlEQAtkMytqZjPN7LCZbTaz/yQz7jUzO2lmhxJsFRPsr2lma8zsSPx/awbuU4hIUrJza+qEChcuwJQp3Rk5sgt58/raVJ84cYQJEzowblxLjh494HGGIiKS2QJ9B3kkcAIoBbQERptZtWTGTnXOFUiwbQIwszzAZ8BEoAjwAfBZfFxEPJATWlMnZGZ07Hg7y5YNomrVMv54dPRk+vWrzebNMR5mJyIimS1gBbKZ5QdaAK845w4555YCs4DW53iqRkAIMMw5d9w5Nxww4OaMzFdE0i4ntaZOqHr1CqxYEUm7drf6Y7t2/cagQdfx1VdD1KZaRCSHCuQd5CrAKefchgSx74Hk7iA3M7O9ZrbezLokiFcDfnBnTgb8IbnzmFknM4sxs5jdu/WrUZHMkNNaUyeUP39e3n67GxMmPEfBgvkAOHXqJB9//ByjRjXj4MFdHmcoIiIZLZAFcgFgf6LYfqBgEmOnAZcDJYCOwKtm9kg6zoNzbqxzrq5zrm7x4uHpzV1EUpBTW1Mn9NBDN7B69RDq1Knsj/3vf18QEVGTX375xrvEREQkwwWyQD4EJK5Qw4GDiQc65350zm11zp1yzi0H3gTuP9fziEhg5OTW1AlVqnQhixf35//+7x5/bP/+rQwbdjOzZ/fm1KlYD7MTEZGMEhLAa20AQszsUufcr/GxGsD6NBzr8M0zJn78c2ZmCaZZXIXvAUAR8cDpFtRbtkzk+PHdhIUVp1y5VjmmNXVCefKEMnBgOxo1qk6HDsPZvfsAzjnmzHmDX35ZRPv2H1G0aFmv0xTJssziKFlyN6VK/UNw8Cmv05Ec5NSpYHbsKMzOncVx7vzuAQe01bSZTcFX7D4G1AS+AK5zzq1PNO4e4FvgH6AeMBN42Tn3QfxqFb8CQ4Ax+KZgvABc6pw7kdL11WpaRDLS1q17adt2KN98s84fy5+/KG3ajKdGjbs9zEwk66pUaQsXXmgULVqK4OBQzCz1g0RS4Zzj1KmT7N27g23bHL/9Vi7JcVm11XRXIB+wE5gMdHHOrTezG8zsUIJxDwMb8U2b+BAY6Jz7ACC+CG4OtMFXQLcHmqdWHIuIZLSLLirKl1++xmuv/YegIN+P08OH9zJ69D1Mnfo0J08eT+UMIrlPePhhSpS4mJCQPCqOJcOYGSEheShR4mLCww+f9/kCOcUC59xefMVt4vgSfA/fnX7/SOIxicavBepkeIIiIucoODiYl19+kBtvrEabNkP46689ACxaNJyNG5fw2GNTKFWqisdZimQtZmrkK5kjo/5s6U+oiEgGuP76akRHD6Vp06v9sT//XEu/frVZuXKCh5mJiMi5CugdZBE5086diwPyYNu6da9y4MAP/vfh4VdRvfobGZpboD5LoK6THsWKhfPJJy8xatQcXnzxfU6ciOX48cO8/34bfv75Kx5+eCR58xZI/UQiIuIp3UEW8Uig2jMnLo4BDhz4gXXrXs2w3AL1WbJDS2sz44knmrJkySAqV77IH1+58kP696/Dn3/+18PsRCQnat68ET16dPM6jRxFBbKIRwLVnjlxcZxaPD25BeqzZKeW1rVqVWTVqihatbrJH9uxYwMDB17DokVvEcgVhETk/D35ZFtKljSGDIk4I75s2TeULGns2ZP2zqFpLWiffLItLVs2TXXc+PEz6NWrf5qvn9iRI0fo2/dlrr66MmXL5qVq1eLcdVcDZsyYnOZzbNnyByVLGv/9b0y688hKVCCLeCQrt2c+19wC9Vmy8tcsKQUL5mPcuKd5772nyZ8/LwCxsSeYOvVJxoy5l8OH93qcoUj2U60alCx59latWuZfO2/evLz11iB2784aLeZPnPAt4FWkSFEKFEiyoXCavPBCZz79dCoREcNYtuxnpk2bz/33t2Lfvtz7M0oFsohHsnJ75nPNLVCfJSt/zVLSuvVNrFoVRY0al/hj33//GRERNdm4camHmYlkP7uSqU2Ti2ekBg1uomzZCgwZ0ifFcStWfMsdd1xD2bJ5ueKKUrzyyv/5i9knn2zL8uWLGTduJCVLGiVLGlu2/JGm65++ozx8+EBq1ChDzZplgLPvSH/++QwaNryKcuXyUaVKUe65pyE7d+5I9rzz5s3i6adfonHjppQrV4GrrqpNu3Zd6NDhCf8Y5xwjRgyiXr1KlCuXj4YNqzN9+r+/vatb1/fzrXHjepQsaTRv3giAuLg4oqL6ULNmWcqUCaNhw+p8+eVnZ1w/MvINatcuT5kyYVSrVponnmjj37dw4VyaNbuBSy8tQpUqRXnwwdvZsOGnNH29zocKZBGPBKo9c3j4VecUT09ugfos2bmldZUqF7NkyUCeeOIuf2zfvj+JimrIF19EEBenjmIiWV1QUBCvvDKADz4Yw++//5bkmG3b/uaRR5pw5ZW1+PrrtQwb9h4zZkwmIuIlAPr2fZO6da/lkUfasW7dNtat28bFF6e9++by5Yv58ccfmDJlLh9//PVZ+3fs2M7jjz/MQw89ytKlP/HZZ9/ywAOtUzxnyZKlWbhwLgcO7E92TP/+vZg06T0GDhzJkiU/8tRTL/HCC4+zYMEcAObNWw3AlClzWbduG+PHzwBg7Ng3GTlyMK+8MpDFi9fRpMm9tGt3H+vW+Z7HmD37E0aNimTgwFGsXPkrH330ObVr/7sa0OHDh+nU6RnmzVvNzJnfEB5eiFatmvn/wZFZtIqFiEcC1Z65evU3znkVi3PNLVCfJbu3tM6bNw9Dh3bkppuuomPHEezbdwjn4pg16xV++WUh7dpNpHDhi1I/kYh45tZb7+TqqxvQv39Pxo6dctb+8eNHUbLkhQwaNIqgoCCqVLmcV14ZwPPPP06PHn0IDy9Enjx5yJfvAkqVKn3O18+bNy9vvjmOsLCwJPfv2LGVkydP0qzZ/ZQtWx6Ayy+/MsVzRkWNpUuXllStWpzLL69OvXrXcccd99Co0W2Ar0gdM2YI06bNp379GwAoX/4S1q5dzbhxI7nttrsoVqwEAEWLFjvjc40aFUnXrs/TosV/AOjR4w1WrvyWUaMiGT16In/9tZlSpS6kUaPGhIaGUqZMOWrW/LfRXbNmLc7I9c03x1OpUjjffbea+vWvP5cv3TlRgSzioZIlGwakuEttSbeknGtugfosgbpOZrr77muoVasibdoMYdky368Kf/llEX371uTRRz/gyiubeJyhiKTk1VcH0aRJfbp2ff6sfRs2/ETdutf6u2sCXH319Zw4cYLff99ItWrJ//YuLapWvTLZ4higWrUa3Hjjrdx445U0atSYG2+8lWbN7qd48RL89dcWrr/+Cv/YZ555mWeeeZlrr72R6OhNrFmzktWrl7FkyUIefLAxrVt3IirqbTZs+JFjx47x8MN3AP92P4yNPUnZshWSzeXgwQNs376Vq69ucEb8mmuu56uvvgDg7rsf4J133qRu3Uu46abbufnmO7j99rv9n/H3339j4MBXWLNmFXv27CIuLo64uDj+/ntLOr56aacpFiIiHihbtgQLFkTw8ssP+tvtHjy4i7feupNPPnmB2NjM/fWhiKRfrVr1aNq0BX36vHjWPudcsi20M6K19gUX5E9xf3BwMNOnz2fatPlcccVVTJr0HvXrX8r//vc9pUtfxMKF//Vvjz7a2X9caGgo9evfwFNP9WD69Pn06NGHCRPGsmXLH8TFxQEwYcLsM47/9tv1TJs2P9Wck/rcp2MXX1yW5ct/ITLybQoWDKd37+e47bY6HD7saxfdunUzdu/eRWTk28ydu4qFC9cSEhLCyZOZ+zNSBbKIiEdCQoJ57bX/MG/eG1x4YRF/fMGCSAYPvp5duzZ5mJ1I1lSixLnFM8vLL/dj5colLFw494z4ZZddQUzMCn9RCbB69VLy5MlDhQqVAAgNzcOpU5n33IGZUa/etbzwQm/mz4+mdOmL+OyzqYSEhFCxYmX/VqRI0WTPUaWK707z4cOHuOyyKwgLC+OvvzafcXzFipX90zjy5MkDcMbnKlgwnNKlL2LVqjMfRl61aqn//OCbNnLbbXfRp89Q5s2L5uef17N69TL27t3Dhg0/8cwzL9Ow4a1UqXI5hw4dJDY2NsO+VsnRFAsREY81alSdmJhhdOjwJnPnfgfA5s3R9O1bi1atxlK37kMeZyiSdaxf73UGPhUrVqZ16068886bZ8TbtevK2LHD6N69K506Pc3mzZvo06cH7dt344ILLgCgXLkKrF27mi1b/iB//gIUKVL0jCkZ5yMmZiXffvsVN910OyVKlGLdurX8/fefZxSkiTVv3oh7732EmjXrUqRIMTZs+JF+/V6mcuXLqFLlcoKDg+na9Xlee+15nHPUr38jhw8fYs2alQQFBdGmTSeKFy9Jvnz5WLRoHmXLViBv3ryEhxfiiSdeYODAV6lY8VJq1KjD9OkTWblyCQsWrAFgypT3iY2NpXbta8ifvwCffTaV0NBQKla8lMKFi1CsWHEmTnyHiy4qy/btf/P66y8QEpL55asKZBEPbdw4hh075gNxQBClSjWmcuXOKR4TiLbR6ZGVW0BnByVKFOLTT3sxfPhsevacwMmTsRw7doB3332Yn376ioceepM8eS7wOk0RSeC5515l6tQPzohdeOHFTJ78Ja+//gI331yT8PDCtGjxH3r27Ocf07Xr83Tr9ig33HAFR48eJSbmd8qVq5AhOYWHF2L16mW8++4IDhz4h4suKsuzz77CAw8kv9rPTTfdzvTpE+jfvyeHDx+iZMnSNGx4G8899yrBwcEA9OjRhxIlSjFqVCTdu3ehYMFwqlWrSbdu3QEICQmhb9/hREW9QWTk69SvfwOffvoNHTs+xaFDB3njje7s2rWDypUvY9y4T6hevWZ8voUZMWIgr732PLGxJ6lS5QrGj59B+fK+ZePGjp1Kz55P0bDhlVxySWVeey2K9u1bJP1BMpDlpm5OdepUditXRnmdhghwujiee1a8VKk7ki2Sk2obDSkXyafbMyfsQBcUFEalSl0zrIANxDVyk5iYX2nVKpJNm/5dt/TCC6/gscemcvHFKT+NLpLV1ar1E5dccrnXaUgO9vvvP7F2bdJ/xjp3tjXOubpJ7kxAc5BFPOK7c5z2OASmbXR6ZKcW0NlB3bqXsnr1UB588AZ/bNu2HxkwoB7ffvu22lSLiGQyFcginok7x3j6BKI9c3ZrAZ0dhIdfwIQJz/L220+QL5/v4ZeTJ48xaVJn3nnnIY4c+cfjDEVEci4VyCKeSe6vX8b+tQxEe+bs2gI6qzMz2rW7jRUroqhWrZw//t130+nbtxabNq30MDsRkZxLBbKIR0qVanxOcQhM2+j0yM4toLODK64oy/Llg+nU6Q5/bM+ePxgy5HrmzRt0xnJSIiJy/gJaIJtZUTObaWaHzWyzmf0nmXEvmNn/zOygmf1uZi8k2v+HmR01s0PxW+qrVItkMZUrd6ZUqTv4969hUIoP6IGvI17iYjgtbaMrVepKWFgJwAgLK5HhD88F4hq5Xb58Ybz1VmcmT+5OoUK+1SxiY08xc+aLvPVWEw4c2JHKGUREJK0CuoqFmU3GVw10AGoCc4DrnHPrE43rDnwF/ABUAuYDLzrnpsTv/wN4zDn31blcX6tYiEhO8McfO2jdegirVv3ij4WHl6Jdu4lcfvmtHmYmkjqtYiGZLVutYmFm+YEWwCvOuUPOuaXALKB14rHOuUHOue+cc7HOuV+Az4AGiceJiORGFSqUYuHCvjz//H3+2IEDOxg+vDGffvoyp06d9DA7EZHsL5BTLKoAp5xzGxLEvgeqpXSQ+Zp13wAk7p3zkZntMrP5ZlYjheM7mVmMmcXs3n0gvbmLiGQpoaEh9OvXhjlzelOyZCEAnHPMndufqKiG7Nmz2eMMRUSyr0AWyAWA/Yli+4GCqRz3Gr48xyeItQQqAOWBRcA8Myuc1MHOubHOubrOubrFi4enI20RkazrtttqERMzjFtu+fc+waZNK+jbtyZr187wMDMRkewrkK2mDwGJK9Rw4GByB5hZN6ANcINzzt+FwDm3LMGw/mb2KL67zLMzLl3JKQLVAjk9baPXrHmSY8f+9L/Pm7csdeqMSPGYZctaAKcSRIJp0OCTFI9Zvrwlzh32vzfLz3XXfZTiMatWtSc2dq//fUhIUa65Zlyy4wP1dVZL67OVLl2EOXN6Exk5k969P+LUqTiOHPmHt99uQcOGXbn//ihCQ/N6naZIjte8eSOqVr2SAQPe8joVOU+BvIO8AQgxs0sTxGpw9tQJAMysPdADuMU591cq53aAZUiWkqOcboF8/PguwHH8+C5++20UO3cuztDr/Ns2+vRyW3Hs2DGXjRvHJHtM4uIY4NixP1mz5slkjzm7OAY4FR9PWuLiGMC5wyxf3jLZYxIXxwCxsXtZtap9kuMD9XUO1HWyo6CgILp3b8GiRf0oX76EP7548SgGDLiGbdt+8jA7kezvySfb0rJl0xTHjB8/g169+qf7GkeOHKFv35e5+urKlC2bl6pVi3PXXQ2YMWNyms+xZcsflCxp/Pe/MenOQwJYIDvf/6FnAG+YWX4zawDcA0xIPNbMWgL9gNucc5sS7StnZg3MLI+Z5Y1fAq44sCzxeUQC1QI5PW2jExfHqcV9EhfHqcU5qzhOLQ6cVRynFg/U11ktrVNXv35VVq8eSvPm9f2xv//+gf7967J8+Xi1qZYc4Z9/PmLDhgqsXx/Ehg0V+OeflH8jltlOnDgBQJEiRSlQILWZo8l74YXOfPrpVCIihrFs2c9Mmzaf++9vxb59Sf/slcwT6EYhXYF8wE5gMtDFObfezG4ws0MJxkUAxYDoBGsdn74VVxAYDewD/gbuAJo45/YE7FNIthG4FsiBaRudVQXq66yW1mlTpEgBpk59kREjHicsLBSAEyeO8OGH7Rk3rhVHj+qBZcm+/vnnI7Zu7cTJk5sBx8mTm9m6tVNAi+TTd5OHDx9IjRplqFmzDOCbYtGjRzf/uM8/n0HDhldRrlw+qlQpyj33NGTnzuTXLJ83bxZPP/0SjRs3pVy5Clx1VW3atetChw5P+Mc45xgxYhD16lWiXLl8NGxYnenT/71JULfuJQA0blyPkiWN5s0bARAXF0dUVB9q1ixLmTJhNGxYnS+//OyM60dGvkHt2uUpUyaMatVK88QTbfz7Fi6cS7NmN3DppUWoUqUoDz54Oxs25NzfTAVyDjLOub1A8yTiS/A9xHf6/SUpnGM9kHzbMJEEwsKKx/86/ux4xgoi6WI4dzSrDNTXOXDfz+zPzHj88SZce+3ltGwZyS+/+GaqRUdP4o8/VvHYY1MoXz7VpUBFspydO3vi3JEzYs4dYefOnhQunPzUsYy2fPliChYsxJQpc5P8zcyOHdt5/PGH6dmzP02btuDw4UOsWZNye/iSJUuzcOFc7r77AcLDCyU5pn//Xsye/TEDB46kUqXLiIlZwXPPdaRw4SLcdttdzJu3mttvv5opU+ZSrVoN8uTJA8DYsW8ycuRgBg8eQ82adZk+fSLt2t3HggVrqF69JrNnf8KoUZG8/fZkLr+8Ort37zwj38OHD9Op0zNUq3YVR48eZejQCFq1asbS/2/v7uNsrPM/jr8+c2PcDqJxl8G4yd26L1OtcVOttkL9bGWRVVpbUtnKmtq1Ekp2Itsv9ZNQ0Wr9UkrbjZKkfRgm/aolTSlNqKh2MYMx+P7+uC7jGDPmjnMx5/18PK4Hvtf3us7nOudyzud8z/dm9cb8x6hIIuPTWyJWuJZALsuy0ZUrNy5VuSe6lOXegLzSlIM3IK805eF6nrWkdel16NCUNWvSGD784vyynTs3M23ahbz99kxgTW4AABIqSURBVCPqciFnnLy8rFKVnyqVK1dm5sy5tGnTnrZtf3bc/u+/305eXh79+v2KxMSmtGnTnqFDbyIhoV6R53z44dmsX59O69Z1ufjiLqSmjmblyuX5+3NycnjiienMmDGHPn0uo0mTZgwcOJihQ3/L3LmPAVCnjjcG4ayz6lCvXn1q1/bet2fNSmPUqLsZOHAwzZu3IjX1fpKTezBrVhoAW7d+Tb16DejV6xecc04inTp1Y8SIo63h/foNpF+/gSQltaRduw7MnDmPrKyvWL9+bfmfzNOQEmSp0MK1BHJZlo3u2vXR45Lh4max8GarKJgMn3gWiwsvXHhcMlzcLBbdu889Lhk+0SwW4XqetaR12VSrVpnZs2/jmWfupEaNKgAcOpTH4sW/Z9as/mRnq4uKnDliYxNLVX6qtG7dnri4uCL3t2vXkZSUS0hJac8NNwxk3rzH+eEH7xewrVuzaNq0ev72yCMPAHDBBSmsW/clS5asYMCAa9m8OZNrr/0Fd931OwAyMzeyf/9+Bg267Jjj589/nC1bNhcZy549u/nuu+2cf/6xa6517/5zMjM3AtC//zXk5u6nW7dmjBkzgpdfXkxu7tExH199tZmbbx7Meec1Jykpnnbt6nH48GG2bQvvF5NwCWsXC5EgJCT0DEsC1aLFzcVO61ZQcVO6Faa4Kd0KU9yUboU50ZRuhQnX8xyux6mIBg1KoVu3lgwdmsb69d6H6SefLGPSpI6MGPEcrVrpeZXTX0LCFLZvH3lMNwuzqiQkTAlrHFWrFv0rHEB0dDSLF79JRsYaVq58k+eee4opU+7hpZfepXXrdqxY8X/5dY+08gLExsaSnNyD5OQe3H57KtOnT2bq1PHcccc9HD7sdeV79tlXaNTo2C8EsbGxxcbsrb1WeFmjRo355z8/47333mbVqreYMOEu0tIm8tpr6VSrVo3rr+9H/fqNSEv7Hxo0aERMTAw//3lb8vIOFPu4ZyK1IIuIRJAWLRqwatVUxozpn1+2a9d2Zszowyuv3MehQwcDjE6keLVqDaFhw9nExjYBjNjYJjRsODus/Y9Lysw477wLGDt2Am++uY769RuydOnzxMTEkJTUIn8LTZALatWqLQA5Odmce25b4uLi2Lr162OOT0pqQePGTQDy+wMfOnR0dqMaNeKpX78h6emrjzl3evrq/POD123k0kuvYNKkGbzxxjo2bdrA2rXv89NPP5KZ+SljxtxLz56X0KpVG7Kz93DwYMV9v1ALsohIhKlUKZZp026kV68OjBgxkx9/3INzh3n11YlkZr7DjTcupHbtc4IOU6RItWoNOS0T4lAZGWtYteotevfuy9ln1+OTTz5k27ZvjklIC7rqql5cffWv6dSpG7Vr1yEzcyMPPHAvLVqcS6tWbYiOjmbUqLu57767cc6RnJySP/gvKiqKYcNGUrduAlWqVOGdd96gceOmVK5cmfj4mtx661geeujPJCW1pGPHrixevIA1a95j+fIPAFi0aD4HDx6kS5fuVKtWnaVLnyc2NpakpJbUqlWbOnXqsmDBkzRs2JjvvtvGxIljiYmpuGmkWpBFRCLU5Zd3IyPjEXr2bJ9f9vnnq5g8uSMff6yFSUXKIz6+JmvXvs+QIVeSnNySCRPu4s47x3PNNUUPKu7duy+LFz/Lddf15aKLWjNu3CiSk3uwePFyoqO98SepqZMYO/Y+Zs1KIyWlHddeeynLlr1AYqI3AVhMTAxTpvyVhQvn0KFDQ4YNGwDAb397O7feOpb77/8DKSntee21F5k79wV+9rNOfry1WLjwKfr370HPnu1ZtuwF5s1bQpMmzYiKimL27OfZuPFjevZsT2rqrYwbN4lKlYrug32ms0gawdy1awu3Zs3DQYchInJaOXToEFOn/i+TJj2f38cRoE+fO7j66oeIja24H4ISfp07f0qzZm2CDkMqsK+++pQPPyz8Hrv5ZvvAOVfsHJcVt21cpBx27HiXrKwF5Ob+QFxcXRITh542A8PKElvBZa2Lmy1DIkt0dDR//ON1pKS0Z9iw6Wzb5q27tGLFTL744j1GjFhEvXotA45SRCR81MVCpIAdO95l8+ZZ/oIUjtzcnWzePIsdO94NOrQyxVYwOQZvOesPPrjtFEcrZ5oePdqRkTGDK644L78sK2s9DzzQhfR0LectIpFDCbJIAVlZCzh8OPeYssOHc8nKCj5BKEtsBZPj4solstWpE8+SJfcyffpNVKrk/ciYm5vNvHnXM3/+cPbvzw44QhGRU08JskgBubmFL5pQVHk4nc6xScVhZowefSXvvfcQLVo0zC9fs+ZpHnywG1u3fhRgdCIip54SZJEC4uLqlqo8nE7n2KTi6dy5OenpDzN48NE+7t9//xlTp3Zn5crHtEy1lJnuHTlVTta9pQRZpIDExKFERR07aj8qKo7ExKKn5gmXssRWcDnr4spFQtWoUYX583/PnDm3U7Wqd+8dPJjLokWjeeKJ/yIn56eAI5QzTV5eLHl5+4IOQyqovLx95OUVv6pgcZQgixSQkNCT5s1HERd3NmDExZ1N8+ajTotZLMoSW9eujx6XDGsWCymtYcP6kJ4+nQ4dmuaXffTRS0ye3Ikvvng/uMDkjJOVlcC3327jwIG9akmWk8Y5x4EDe/n2221kZSWU+3yaB1lEREps//4DpKbOZ9asf+SXRUVFc+WVE7nsslSioqIDjE7OFPHxu0lM3EFsbF7QoUgFkpcXS1ZWArt3xxdZp6TzICtBFhGRUlu6dA0jR/43//730VktWre+mBtueJaaNRsEGJmISNFKmiCri4WIiJTagAHJrFs3gwsvPLpa1aZNbzN5ckc2bHg9wMhERMpPCbKIiJRJYuLZvPXWZO655xrMDIA9e3by6KO/5IUX/sDBgwcCjlBEpGzCmiCb2Vlm9qKZ5ZjZ12Y2uIh6ZmYPmdmP/jbNjrz7evs7mdkHZrbX/7NT+K5CRESOiImJZuLEIbz++kTq16+dX758+V9IS+vBzp1fBhidiEjZhLsF+THgAFAPGAI8bmbtCqk3ErgK6Ah0AK4EfgdgZpWApcACoDbwNLDULxcRkQD07t2BjIwZ9O3bJb9sy5a1TJnSmYyMvwcYmYhI6YUtQTazasBAYLxzLts5txp4Gbi+kOq/AR52zm11zm0DHgaG+/t6ATHAI865XOfcXwED+pziSxARkRNISKjF0qV/YurU4cTEeLNZ7N+/mzlzrmPBgpEcOLA34AhFREomJoyP1Qo45JzLDCn7CChsAtd2/r7Qeu1C9n3sjp1+42O//LiRIWY2Eq9FGiC3UqWr/lW28KUCqAtoTebIpnsgIKtXP8nq1U8GHQboHoh0ev3l3JJUCmeCXB3YVaBsF1CjBHV3AdX9fsilOQ/OudnAbAAzyyjJ1B5SMen1F90Donsgsun1FzPLKEm9cPZBzgYKztwcD+wpQd14INtvNS7NeURERERESiWcCXImEGNmLUPKOgIbCqm7wd9XWL0NQIfQWS3wBvIVdh4RERERkVIJW4LsnMsBlgD3m1k1M7sIGAA8W0j1Z4A7zayRmTUE7gLm+/tWAoeA280szsxG++UrShDG7HJcgpz59PqL7gHRPRDZ9PpLie6BsC41bWZnAXOBS4EfgVTn3HNm1gN4zTlX3a9nwEPATf6hc4BxRwbmmVlnv6wt8Ckwwjn3YdguREREREQqrLAmyCIiIiIipzstNS0iIiIiEkIJsoiIiIhIiIhIkM3sLDN70cxyzOxrMxscdEwSPmY22swyzCzXzOYHHY+Elz+Y9yn///4eM/vQzH4ZdFwSXma2wMy+NbPdZpZpZjcVf5RUNGbW0sz2m9mCoGOR8DKzlf5rn+1vn52ofkQkyMBjwAGgHjAEeNzM2p34EKlAtgOT8QaISuSJAb7BW7WzJjAe+LuZNQ0wJgm/B4Gmzrl4oD8w2cy6BhyThN9jwLqgg5DAjHbOVfe3E66oV+ETZDOrBgwExjvnsp1zq4GXgeuDjUzCxTm3xDn3Et7MKRJhnHM5zrn7nHNbnHOHnXPLgK8AJUcRxDm3wTmXe+Sf/tY8wJAkzMxsEPAf4O2gY5HTX4VPkIFWwCHnXGZI2UeAWpBFIpCZ1cN7X9DiQhHGzGaZ2V5gE/At8I+AQ5IwMbN44H68dRUkcj1oZj+Y2ftm1utEFSMhQa4O7CpQtguoEUAsIhIgM4sFFgJPO+c2BR2PhJdzbhTee38PvIWrck98hFQgk4CnnHPfBB2IBGYckAQ0wlss5BUzK/JXpEhIkLOB+AJl8cCeAGIRkYCYWRTeyp0HgNHFVJcKyjl3yO9qdw5wS9DxyKlnZp2AS4AZQcciwXHOpTvn9jjncp1zTwPvA5cXVT8mfKEFJhOIMbOWzrnP/bKO6OdVkYjhr875FN5A3cudc3kBhyTBi0F9kCNFL6ApkOW9FVAdiDazts65LgHGJcFygBW1s8K3IDvncvB+SrvfzKqZ2UXAALyWJIkAZhZjZpWBaLw3xcpmFglfDuWox4E2QD/n3L6gg5HwMrMEMxtkZtXNLNrM+gK/BlYEHZuExWy8L0Od/O0J4FWgb5BBSfiYWS0z63vk89/MhgApwBtFHVPhE2TfKKAKsAP4G3CLc04tyJHjT8A+IBUY6v/9T4FGJGFjZk2A3+F9MH4XMgfmkIBDk/BxeN0ptgL/BtKAMc65pYFGJWHhnNvrnPvuyIbX9XK/c25n0LFJ2MTiTfe6E/gBuA24yjlX5FzI5pwLU2wiIiIiIqe/SGlBFhEREREpESXIIiIiIiIhlCCLiIiIiIRQgiwiIiIiEkIJsoiIiIhICCXIIiIiIiIhlCCLiJzhzGy4mWUXU2eLmd0drphOxMyampkzs25BxyIiUhglyCIiJ4GZzfeTPmdmeWb2pZmlmVm1Up5j2amMM9wq4jWJSMWn5XZFRE6et4Dr8VZt6gHMAarhreImIiJnCLUgi4icPLn+crbfOOeeAxYCVx3ZaWZtzexVM9tjZjvM7G9mVt/fdx/wG+CKkJboXv6+qWb2mZnt87tKTDOzyuUJ1MxqmtlsP449ZvZuaJeHI902zOxiM/uXmeWY2Ttm1qzAee4xs+/9us+Y2QQz21LcNfmamNlyM9trZhvN7NLyXJOIyMmiBFlE5NTZh9eajJk1AFYB/wLOBy4BqgMvm1kUkAb8Ha8VuoG//dM/Tw5wI9AGGAUMAv5Y1qDMzIBXgUbAlUBnP7YVfpxHxAH3+I99AVALeCLkPIOACX4sXYBPgTtDjj/RNQFMAf4KdATWAYvMrHpZr0tE5GRRFwsRkVPAzM4HBgNv+0W3AB8558aF1BkG/AR0c86tNbN9+K3Qoedyzk0K+ecWM3sAuBsYX8bwegOdgLOdc/v8svFm1g+vi8g0vywGuNU595kfbxowz8yinHOHgTuA+c65OX79B82sN9DKjzu7sGvy8nMAZjjnXvHL7gWG+XGtLuN1iYicFEqQRUROnsv82SRi8FqOlwK3+fu6AilFzDbRHFhb1EnN7FfAGKAFXqtztL+VVVegKrAzJFkFqOzHckTukeTYtx3vumrhJfatgScLnDsdP0EugY8LnBsgoYTHioicMkqQRUROnlXASCAP2O6cywvZF4XXraGwqda+L+qEZpYMLAImAr8H/gP0x+u+UFZR/mP2KGTf7pC/Hyywz4UcX7CsLPKfH+ec85N1df0TkcApQRYROXn2Oue+KGLfeuBa4OsCiXOoAxzfMnwRsC20m4WZNSlnnOuBesBh59yX5TjPJrz+1PNCys4vUKewaxIROa3pm7qISHg8BtQEnjez7maWZGaX+DNJ1PDrbAHam9m5ZlbXzGKBTKCRmQ3xj7kF+HU5Y3kLeB9Yama/NLNmZnaBmU00s8JalYsyExhuZjeaWUsz+wPQnWNblQu7JhGR05oSZBGRMHDObcdrDT4MvA5swEuac/0NvP68nwIZwE7gIn8Q21+AR/D67F4K/LmcsTjgcmCF/5if4c02cS5H+wKX5DyLgEnAVOBDoD3eLBf7Q6odd03liV1EJBzMe58UEREpPzN7EYhxzvULOhYRkbJSH2QRESkTM6uKN33d63gD+gYCA/w/RUTOWGpBFhGRMjGzKsAreAuNVAE+B6Y55xYGGpiISDkpQRYRERERCaFBeiIiIiIiIZQgi4iIiIiEUIIsIiIiIhJCCbKIiIiISAglyCIiIiIiIf4fC2/gUzLsMXUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff94bf961d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1] \n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"Petal length\", fontsize=14)\n",
    "plt.ylabel(\"Petal width\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions\n",
    "# 激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z)) # 逻辑函数\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z) # relu函数\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps) # 求导"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure activation_functions_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd8FGX+wPHPd9M7hJBAiBjpTYoEQRHNiQUsgIeeIIp6IipnF+sPPbw7y9lOsR3cqXCCeCoKVgRPIiBKUQlFMBRpAVNoIaRnn98fs6mkZ3Y3Id/36zWv3Z155plnhzCz33maGGNQSimllFJKqbpweLsASimllFJKqeZDAwillFJKKaVUnWkAoZRSSimllKozDSCUUkoppZRSdaYBhFJKKaWUUqrONIBQSimllFJK1ZkGEKrJEpFdIjLVA8eZLiKbPHAch4jMFJGDImJEJNHdx6ylPLNF5FNvlkEppZoTEblBRLI9dCwjIld64lhK1ZfoPBDKDiIyAFgHfG+MGVrPfacDVxpj+lRa3xY4bozJsamM8cCvwCBjzLpy60OBAGPMQTuOU8PxLwM+BBKBncAhY0yBO4/pOm4isAxoa4zJLLc+AusacMTdZVBKKU8QkdnA9a6PRcBhYDPwATDLGFPYyPyDgDBjTHpj8qmU52wgyhhzWaX17YDDxph8u46llF20BkLZ5WbgNaCPiPS0I0NjTIZdwUMtx8l2d/Dg0gU4YIxZZYz5zRPBQ02MMUc1eFBKnYS+AtoD8cBFwCfA48AKEQlpaKYi4meMybUzeKiJ6z6hwYNqkjSAUI3meiJzDfAvrKc8N1WRJlZE5rma7+SIyHoR+Z2I3AD8Gejtqq41rnUVmjCJyHwRWVApT4eI7BWRe1yfR4jIChE5LCKHROTLSsHMr67Xta7jJLn2q9CEyZXvo66880Vko4iMLrc93rX/WBFZ6vo+P4vIhTWco9nAP4COrn13udYnicgrldOWb1rkSvOaiDwpIpkiki4iz4mIo1waf9f23a4y7xSRO121LstcyTJcx55dzXECRORFEUkTkTwR+V5Ezim3PdG1/3ARWe363utE5IxyaSJE5G1XGfNc5bi7uvOilFJukO/68Z1qjFlvjHkBq+b3DOABKL1m/l1E9onIcRFZKyIXl2RQ7np3iYisEZEC4OLyTZhEpJsrzenlDy4ik13Xaj8R8RGRN0TkVxHJFZFtIvJAyfXbVQN/PXBpuXtgomtbaRMmEflORJ6vdJxwV55X1PE7+YnIDBHZ77pP7BWRp20986rF0ABC2eFKYLcxZgPwNjBRRPxKNor1xOcbrKdBVwCnA39xbf4v8DzwC9YTo/audZXNxbrAtiq37jxX+vmuzyHAi8CZWDeLo8AnIuLv2n6m63WEa7/fV/N97gLuBx50lfUj4EMR6V8p3RPADKAfsBZ4V6zmUNXl+Rdgn+vYg6pJV50JWNXxZwO3A3cDV5fbPgeYCNwL9MQK4o4Ae4GxrjS9Xce+q5pjPOPK84/AAGAjsFhE2ldK9xTwENbN+CAwT0TEte1vWOfsMqCHK6/Uen5XpZSylTFmE7CYsuvhW1j3kGuwrllzsO4X/Srt+ndgGtb1bHWlPFOwmu5OqLTPBOC/ruZSDqxr4B+wrs3/BzwC3OhK+xzwHmW1Ju2BVVV8hbnAuPIPjlzfJRf4rI7f6U6se/A4oCvW9f6XKo6lVO2MMbro0qgFKziY6novwC5gbLntNwPHsNp4VrX/dGBTFet3lcvXF0gHbiq3/d/AlzWUKwQoBs5xfY4HDJBQ0/GxLvaPVUqTBMytlM8t5bZ3cK07p4byTAV2VZHvK5XWzQY+rZTmu0pplgL/dr3v6jr2iGqOm+jaHlXdcVznqgCYWG67D7AD+FulfC4ul2aoa12c6/PHwFve/pvURRddWuZS+fpZadvTQA7QGXACHSttXwi85npfcr0bWynNDUB2uc93Absp61N6iivvs2oo49PAV7WV2XX8K13v27iu0cPLbf8KmOl6X5fvNAP4X0lZddGlMYvWQKhGEZEuWD8i3wEwxhhgHjCpXLIBwAZTrgNvfRljirBqJia4jhuA9fRlbrmydBaRd0Rkh4hkAWlYT3861uP7hAOxwLeVNq0EelVat6Hc+/2u1+i6HqueNlT6vL/csQZg3TiW0XCdAT/KfW9jTDHwHfX73q8DfxCRZFczq/MaUSallLKTYP0oP8P1/mcRyS5ZgEuxroXlraNm87HuGcNcn68Bdhpjvis9qMitruaeGa7j3EM97ksAxuqn9yVl98D2wO8ouwfW5TvNBvoDKSLyqohcWqlGQ6k68/V2AVSzNwnrSfWeslYsCICInGKM2Vvy2QZzgVUi0gEYDPhjNS8q8QlW7cEtrtci4GdXuvqqaniyyutKR/MwxhjX96/vxdjJiefHr4p0lUcOMeWOZcf5LcmjXt+73DYHgDHmCxE5FRgJDAc+E5H3jTE3opRS3tULawQ8B9a1axAnXltzK30+XlOGxph0EfkK64f9ctfrvJLtInI1VtPaqVhNk7KAP2E1JaqvucAsEZkCjMdqorrSta3W72SM+dHVL24EcD5WE6dkEbnQGONsQHlUC6aRp2owEfHF6vz1MNZTjZKlH9ZT6pIfjT8CfUUkqpqsCrCCkBoZY1ZjNakZj3WRXmiMKenM1garfemTxpivjDFbgDAqBsklox5VeyxjTBbWU/VzKm06BysYsVsGVpvX8iq3wa3Nj1j/l39XzfZavzew3ZWufKdpH+As6vm9jTGZxpi3jTE3YPXFuN5VY6SUUl4hIn2wfjh/APyE9dCknTFme6WlIX225gJXichArL4Hc8ttOwdYbYx5xRjzozFmOyfWctTpHggscr1ehitQcdX6U9fvZIw5Zox53xhzG1btxPlYIwQqVS9aA6Ea41IgCviXqTQMqoi8C9wmIn/Dat70ELBQRB7G6kh8OnDMGLMMq6/Dqa7RfPa41lc3dF1J86h4Kj7BOQxkAjeLyF6sPgnPYtVClEjHehJzsVijIOUZY45WcYxngb+IyDbgB+BarOrpgbWdkAb4GnhRREZhdWa7BasN7a66ZmCM2SYi7wH/FpG7sAKKOCDeGPM2Vvtcg9UJ/RMgtyTwKpfHcRF5HXhaRDKxRqy6B4jBGp63TkTkL67jb8a6vvweqzpfhyJUSnlKgFhzKDiAtli1oY9gXc+fc13v5gGzReQ+rGtWJK45eowxH9bzeB8B/wTeANYYY7aV25YC3CAiI7Ee1IzD6uh8uFyaXcBIEemONTDFUVPFfBXGmDwR+RCrU3c/rHtTybaU2r6TiNwLHADWY9VSXINVI7Kvnt9XKa2BUI1yE7CscvDg8j5wKnCBMeY41gUzFauZ0WasMblLnpwsAD7H6tyVgVXDUJ25QHesEZaWlqx0Vb9eDfQFNgGvAo8C+eXSFGGNQjEJq5ZhEVWbgRVEPOPK6wqsjnTrayhXQ71ZbvkWyKZis6y6mogVqM0AtmK1dY0AcD19+jPWqFFpwCtVZ8GDWKOBvIV1g+mL1TH7QD3Kke86TjLW9wkDLq/fV1FKqUa5AOuH8h6s+8oorHvOua77EVg15G9hXee3Ap8C52I9cKkXY81X9BHWj/q5lTbPxLquvoM1Wl881siD5f0L2ILV3yIDq19hdd52HedHV017ebV9p2NYIwyuwQow+gMjjQfmW1InH52JWimllFJKKVVnWgOhlFJKKaWUqjMNIJRSSrmNiNzuGsIyX1yzoFeT7noR+UFEslwz6T7jGqhBKaVUE6MBhFJKKXfajzVD+Zu1pAvGmmE9CmuY5uFYQ18qpZRqYvTpjlJKKbcpGdFGRBKwRgerLt3r5T6mukaUqW5oYqWUUl7UrAKIqKgoEx8f7+1icPz4cUJCQrxdjCZDz0dFTf18mEJD4aFCfCN8cQS6vxKyqZ8PT2sq5+OHH37INMa09XY5anAu1ohtVRKRycBkgKCgoIGnnHKKp8pVLafTicOhFfsl9HxUpOejIj0fFTWV85GSklKne0OzCiDi4+NZt662WeXdLykpicTERG8Xo8nQ81GRno+K9HxU1FTOh4jUe7hKTxGRG4EErCGXq2SMmQXMAkhISDB6b2h69HxUpOejIj0fFTWV81HXe0OzCiCUUo1jjEFEvF0MpaolImOAp7HmkMn0dnmUUkqdyPt1JUopjyg8UsiaHmvYOW0nOv+LaopEZATWpFqXG2M2ers8SimlqqY1EEq1EJkLMslNySXruyythVAe4xqK1RfwAXxEJBAocs0MXz7d+cA84ApjzBrPl1QppVRdaQ2EUi1E2rw0AGImxHi5JKqFmQbkAg8B17reTxORjiKSLSIdXekeBSKAz13rs0XkC+8UWSmlVE20BkKpFiBvXx5Hko4gAULbsU154B11sjHGTAemV7M5tFw6HbJVKaWaCa2BUKoFSJ+fDgbaXNYG3wh9bqCUUkqphtMAQqkWQJsvKaWUUsouGkAodZI7vvk4x5OP49vKlzaXtPF2cZRSSinVzGkAodRJrqT2oe1VbXEE6H95pZRSSjWO/ppQ6iRmnEabLymllFLKVhpAKHUSO/rtUfL35BNwSgARwyK8XRyllFJKnQQ0gFDqJFZS+xA9Phpx6ORxSimllGo8DSCUOkk5C5xkvJcBaPMlpZRSStlHB4RX6iQlPkLPeT05knSE0L6hte+glFJKKVUHttZAiMjtIrJORPJFZHYtae8Rkd9E5KiIvCkiAXaWRamWTnyENiPb0Pnvnb1dFKWUUkqdROxuwrQf+BvwZk2JRORi4CFgOBAPdAIet7ksSimllFJKKZvZ2oTJGPMhgIgkAHE1JL0eeMMYs9mV/q/APKygQqlmLWT7dvj2WzDGa2VI2xBFxqYo4s7eT6v4LK+VA+DUX3+FlSu9Wgaw/jkKin3ILfQlr8iXIqeDYuOgyOmgqFis1yqW0jROoai4bJ0xYBCMAaeR0vcGsT5Xsz0j8yBb2nyOMYIBjHGld703uNKbip3eK/81nbC9UgJDzduVUkqphvJWH4jewKJyn5OBGBFpY4w56KUyKWWLHs88A9u2ebUMv/EMh+lB5JY5tOJTr5bltEbsm0sgGbQlnWgyaMtRIsgivNolm1DyCCSPQHIJKn1vLUG2fSellFKqJfNWABEKHC33ueR9GFAhgBCRycBkgJiYGJKSkjxRvhplZ2c3iXI0FXo+KhqUZT3xTx0zhqJQ73ReDs3Zjuxyktc5mt0B13qlDCUKCgrw9/c/Yf2hvFD2Zrdlf04bUrOj2Hc8iv05bcjIjeBgXjgH88I5XmTvj35/RyEBPoX4O4rwdRTh63DiI8X4ihMfRzE+4sTX9Vr+femrFONwWNus5/sGh+u9iOGYo4BcRwFFjmIKxUmRFFPoKKJQigk1vvQuaIVxFlPk62RJyF6KpZgiRzFFUoxTnCAGxMnw4x04rSgUAX4MzOCHQGs0LaRiNUKAcXDD0R6ln+dF/EK2o7BcirL0A/KiGJJnjca1x+8Yn6239dQqpZRqQbwVQGQD4eU+l7w/VjmhMWYWMAsgISHBJCYmur1wtUlKSqIplKOp0PNRUZ7rtcOLL8Kpp3q1LE3BokUrCQo6h82bYcuWsuVgHeoa/fwgOhratrWW1q0hPPzEJSLCeg0JgaAgCAwsey1ZAgLA4fAD/Ko9njGGQ7mHAGgT3AaAXw//ypzkOaQfTycjJ8N6PZ5BRk4Gh3MPs+2ObZzW2qpnuer9q/jg5w+qzDsxPpH51y8jKSmJAUMG0OrvF1fY7hAHIX4hhPiHcMel/2R0j0sAeH/z+8xJTiLAN4BA30ACfAII8LHehweE8/jvLi3NY9iGI+QV5ZWm9ffxL13iW8XTqXUnALILsgnTYSuUUko1kLcCiM1AP+A91+d+QJo2X1InBafTenV4Z5oVYwwi3pk0rqgIfvwRVq8uW7ZvP6fKtGFh0KmTFWOdeip07GgtHTpYwUJ0tBUY2PVVjDFQrl/AnPVz+DnjZ/Yd28e+LGtJzUolvzifuwbfxYsjXgTgQPYBHv+m+jEejuaXVaYmtE8gpzCHiIAIWgW2qrCc1qqsMVdYQBg/T/mZEP+Q0qAhwCegyn+3q3pfxVW9r6rTd5zQd0Kd0oX667C+SimlGs7WAEJEfF15+gA+IhIIFBljiiol/Q8wW0TmAQeAacBsO8uilLdISW9VLwQQOb/ksGHkBmJviaXjgx09csxdu+CLL2DJEvj6a8iq1Gc7IKCYhAQf+vaFnj3LlthY+4KDEkXOIvYc3cOOQzvYcXgH2w9tZ8fhHew4tIN9WfvIuD8DH4cPAK+ve53VqatPyCMiIAIpF2h0bt2ZacOmER0STduQtkSHRFvvg9vSOqg1/j5lzbMePOdBHuTBWsvpEAc92/a04RsrpZRSnmd3DcQ04M/lPl8LPC4ibwI/A72MMXuMMYtF5BlgGRAELKi0n1LNlxdrINLmpZH3ax45v+S49Tg7d8IHH8D778O6dRW3de0KQ4fC4MHWcvDgSi644Dxbj19QXEDKwRQ2p2+mc2RnEmITAPjg5w8Yv2B8tfvtzdpLfKt4AG4acBOjuo8iLjyudOkQ1oEQ/5AK+8SExvDX8/9qa/mVUkqp5szuYVynA9Or2VyhztwY8wLwgp3HV6opKK2B8PHx6HGNMaTNSwMg5toY2/PPy4MPP4RZs+Cbb8rWh4TAyJFw8cVw4YUndvtISmr8+KHf7vmW7/d9zw8HfmD9b+tJOZhCsSkG4M4z7ywNILpEdiEuPI7OrTtbS6T12iWyC50jO9MqsFVpnjcPvLnR5VJKKaVaIm/1gVDqpCXF1g9bT9dAZH2fRd7OPPxj/Wl1Xqvad6ij336Df/wD/v1vOGT1LyY4GMaMgSuvhBEjrA7LdigoLuCH/T/w/b7vuXPwnaXNje5bcl+F5kaC0Ll1Z3pH96Z/u/6l6xNiE9h7z157CqOUUkqpKmkAoZTdvNQHoqT2IXp8NOLT+M4Fu3fDM8/AG29Afr61bsAAuOUWGD/eGvWosbLys1i1dxUr96xk5Z6VrE5dTV6RNY7ViC4jSvsJXNnrSvq368/A9gM5o/0Z9Gzbk2C/4MYXQCmllFL1pgGEUjaTkj4QHmzC5Cx0kvFfa66AmAmNa7505Aj89a/w8stQ6JpSYMwYeOghq09DY5QfIWpj2kb6z+yP0zgrpOnVthdDTxmKn0/ZcKtTz57auAMrpZRSyjYaQChlNy90oj689DCFmYUE9wwmtH/DhugsLrZqG6ZNg4wMa4Ska66Bhx+GPn0aVi5jDHty9jBj9Qy+3PElAT4BfHj1hwD0iOpBeEA4PaN6ck7Hczin4zmcfcrZRAVHNexgSimllPIIDSCUspk3hnEt7Tw9IaZBc0Bs2wYTJ8L331ufzzkHXnwRBg6sf1mKncWs3LOShVsXsvCXhew6sqt0W5BvEPlF+QT4BuDn40f61PQKNQ3q5CMitwM3AKcD840xN9SQ9h7gQcpG57vNGJPvgWIqpZSqBw0glLKbh2sgirKLyFyYCUD0NdH12tfphNdegwcegNxcaxK3F16Aq65q+BwN//rxX9z22W2lnyP8Irik+yVc3PliLup8EQG+ZVMga/DQIuwH/gZcjBUYVElELgYeAs537fMR8LhrnVJKqSZEAwilbObpYVwzF2bizHESfnY4QafVfTikzEyYMMGaAA7guutgxgxoVccBnJzGyco9K3ln4zt0at2JB4Y+AMBl3S7jhe9eYEyPMYzpMYa87Xmc/7vz6/u11EnCGPMhgIgkAHE1JL0eeMMYs9mV/q/APDSAUM3Y7t3WKHY//wyHD/fltlP202vrPuJuiSXuLuu/w5GVR0i5NaVe+cbefOL+EUMj6D6zOwCFhwr56dyf6pVnVfv7tfZjwIoBpWmSL0wm/0DdKwWr27/fkn6l67bdtY3D/ztcr7L2W9KPgNiACvt3ebELkRdEArB/1n72zdhXrzyr2r+q81wf9fp3Og5rQtbUmqcn/p3qQgMIpWzm6WFc0+elA/Wb+2HDBhg92ppFOioKZs6E3/++bvv+nPEzs9fPZv6m+ezLsi7Q8a3iuf/s+xER4sLjSLmj7CKbtCOpzuVSLVpvYFG5z8lAjIi0McYcrJxYRCYDkwFiYmJISkrySCFrkp2d3STK0VS05PNhDMyb15HZs+MpLi65F0Ty9brddCKHZR+lckq/7dbqNcDm+uW/fe12tidV3D8nOIcDSQesdUfqn2eV+0dQ8d9wI5BWj0yr2f+75d+RHer6+/ip/mX9bvl30M71wbX/hlUbyn7VNuCcVrV/Vee5Pur775RD7ZPAeuTfqQ40gFDKbh7sA1GQXsChpYcQX6HtVW3rtM+CBVZ/h5wcGDQIPvrIarpUm6RdSfzf1//Hqr2rStedGnEq15x+DeP7jG9Q3wulygkFjpb7XPI+DDghgDDGzAJmASQkJJjExER3l69WSUlJNIVyNBUt9XwYA/ffbw1K4XBYNb3jx8PatZtZu6QrN37XhmMr/HnlN3/GjYOigUXkXZJXr2P4t/XHP8YfKNvfJ9SHoHirFtpZ5CRnY+0/Rsuran/xEUJ6hpSmyfkmB2ehs7osTlDd/sHdglm+ajmJiYnkzc2jKKuoXmUN7haMw9+6x5bsH3hKIL4R1s/agl4FFNxdUK88q9q/qvNcH/X5d1q3dh0JgxJqzdMT/051oQGEUnYyxqOdqH1b+9JnYR9ytuTgH+Vfa/pnn7X6O4DVZGnWLAgMrD59SYdngMLiQlbtXUWYfxjj+4zn+v7Xc1bcWRo4KLtkA+VnFyl5f8wLZVGqwZ5/3lr8/GD+fBg7FvJ25xHyVQa3T2vHCyvb8NRTVmARGwvnnutLaJ+GjZ4H4Bt24v4OX0ej8qxu/+DujZt/p6r9AzvWcBOqg6r294/2xz+69ntidarav6rzXB+1/jtlUu/83fXvVBcaQChlJw93oHb4OYi6LAouqzmdMfD449YiYgUS995bdUfpImcRi7YuYsaaGUQFR7HgDwsAGN5pOHOvmMvoHqMJ9W/4RVSpamwG+gHvuT73A9Kqar6kVFP100/wyCPW+3ffLWsaenzTcXgR9qek8uRnbSgshOeesx7kJCfXve+ZUk2FBhBK2ckLc0DUxhh48EEraHA4YPZs66ZV2eHcw/zrx3/x6tpX2XN0DwCtAltxLP8YYQFhOMTBhL4TPFt41eyJiC/WvcYH8BGRQKDIGFO5zcJ/gNkiMg84AEwDZnuyrEo1Rn6+VatQWAh/+lPFfmWm2KqZFof11ObJJ+Gbb2DtWrjjDnj7bW+UWKmGazq/cpQ6GXgwgNjzzB42X7WZrHVZ1aYxBu6+2woefH2tJ2KVg4eM4xncv+R+Or7YkQe/epA9R/fQNbIrM0bMYPfduwkLCHPzN1EnuWlALtZoSte63k8TkY4iki0iHQGMMYuBZ4BlwG7X8mfvFFmp+nv9ddiyBbp3t6655ZUEELgG5/Pzg3nzICgI5s6F1as9W1alGksDCKXsVBJAeGAI1wNvHiDjgwyKDlXf+ezJJ62hWf394cMPrfkdKsspzOHF1S+SXZDNBZ0u4PNrPmfr7Vu5Y/AdhAeEn7iDUvVgjJlujJFKy3RjzB5jTKgxZk+5tC8YY2KMMeHGmBt1EjnVXBw5An/9q/X+ueeswKC80hoIn7J2o127Wg94wOqbVtJ9TqnmQAMIpezkwSFc+y3tR+cXOtPq/Kobz86ZA9OmWf0c5s+Hyy+31v+S+QsPf/UwxnW3OrXVqbw04iXW3byOpdctZWTXkThELw1KKVVXTz8Nhw7BeefBpZdWkcD1bKl8AAFW89I2bWD5cvj0U/eXUym76K8EpezkwSZMgacEcso9p+DwPfFYS5bApEnW+xkzrLa42w9t5/qF19PrtV48/e3TLNy6sDT9lEFTGBg70O1lVkqpk82RI/Dqq9b7Z56penCK0iZMlS7XERHWgx6AJ57QWgjVfGgAoZSdPNCEyThNae1BVTZutIYNLCqyqsUvv3Y3kz6eRI9XevCf5P/gEAeTz5hMQmzt400rpZSq2axZkJ0Nw4fDmWdWnaaqJkwlJk+GyEirH8SqVSdsVqpJ0gBCKTt5oAnT4aWHWdNtDan/TD1hW1aWFTxkZ8M110DwiL/S7ZVuvPHTGwDc2P9GUm5PYeblMzkl4hS3lVEppVqCggJ46SXr/X331ZDQdWuoKoAIDoYpU6z3zz1nb/mUchcNIJSykweaMKXNSyN3ey6F6YUV1htjNVvatg369oV//xuC/AMoKC5gfJ/xbPnTFt4c/SantT7NbWVTSqmW5L33YP9+6NULRoyoPp1xVt2EqcSf/mQNdrFoEWzfbn85lbKbBhBK2cnNTZiKc4rJ/CgTgOhroitse/llw/vvQ2BIAR98YI0CcufgO1l38zreGfsOXdt0dUuZlFKqpZo503q9++6q+z6UqKkJE0C7dlatsTHWwx+lmjoNIJSyk5ubMGV+nElxdjHhQ8IJ7lI2Vf07X+zg7nut4Vx9x0wmKu4wAIG+gdo5Wiml3GDrVli5EkJCYNy4WhLX0ISpxM03W6+zZ1uT0SnVlGkAoZSd3NyEKX1eOgDRE6zahyN5R/jTwgeYMAFMsR+B58zk+XvO1snflFLKzd5803odNw7CarnkljRhqimAOOss6NED0tLg88/tKqVS7qEBhFJ2cmMAUZBZwKHFh8AHoq6KYs76OXR/pTuvPd0BDncm6rR97Pr0D0weOBlfh6/tx1dKKWUpLLTm2gG46aba07dKbAVTyx7+VEWkLK833rChkEq5kQYQStnJjX0gMt7LwBQZIi+KxK+tHy+ufpH0zd1h9V34+hqWfhhHTERr24+rlFKqosWLIT0devaEIUNqTx/SMwQuhVbnVD3xZ4mJE8HXFz77zMpfqaZKAwil7OTGPhAH5h4AIGZCDD4OH1743T+J/t9nADz8sNC/v+2HVEopVYX5863X666rufN0fUVHw8UXW8+i3n/fvnyVspsGEErZyU1NmJYkLSH7u2wKAwppM7oNAJ/8czDpe8M4/fSymUyVUkq51/Hj1nCrUIfO0y7HfjoGi+Do90drTTt+vPVaEqQo1RRpAKGUnWxuwpR+i08rAAAgAElEQVRxPIPrPrqO//z1PwAk900m1y+XtWvhxRetw7z1ljV+uFJKKff75BPIybGaLp1Wx2l1Di89DC9C5oLMWtOOHm0Nw/3tt7BnTyMLq5SbaAChlJ1sasJkjOHt5Lfp+WpP5ibP5aKNFwFw3aPXEeIXxp13WuOF33svDNRRWpVSymNKagZKagrqIrR/KFwO4UPCa08bCpdfbr1/990GFFApD9AAQik72dCEqbC4kMvmX8bEhRM5mHuQa3yuoWNGR/za+tF2RFveeQe+/96aeOjRR20qt1JKqVodO2Z1oBaBq66q+36RF0XCvdB2bNs6pS8JTrQfhGqqdKxHpexkQwDh5+NHfEQ8rQNb84+L/8G1Pa7lcN/DFB4qJCffwYMPWumeeqr2sceVUkrZZ/FiKCiAoUOhfXv3Heeii6xmTOvWQWoqdOjgvmMp1RBaA6GUnRrYB+Jw7mE2pW8q/fzMhc+wacomru9/PT6BPkSNjqL9je158knYvx/OPNMa7k8ppZTnlHSeHj26fvvl7c2DLZC/P79O6YODrSAC4OOP63cspTxBAwil7NSAPhBLdyzl9NdPZ9T8URzLPwZAiH8IsWGxFdLt2AHPP2+9nzHDbZNdK6WUqkJhoTU/A9Q/gNg/cz9MgQNvHKjzPiXHKAlalGpK9CeIUnaqRxOmnMIcbv/8di6aexGpx1KJCY3haH7FIf5Sbk9h87jNHN9ynIcftqrOJ06EwYPdUXillFLVWbECjhyBHj2gW7d67uy6NYhP3SeNuOwy61by9deQlVXP4ynlZhpAKGWnOjZhWr1vNQNmDuDVta/i6/DlifOfYMWNK4gLjyvLqsBJ2tw0Mv6bwdZfhPffh4AAeOIJd34BpZRSVWlo8yUAU2ysN/X41dW2LZx9tlXzsXhx/Y+plDtpAKGUnerQhOn5Vc8z9M2hpBxMoXfb3qyZtIZHhj2Cr6PimAYOfwcJ6xPo9q9uPP5mMABTpkBcXFW5KqWUchdj7Akg6lMDUf5Y2oxJNTUaQChlpzo0YYpvFY/TOLnvrPtYN3kdA9oPqDZtUHwQe0+P5ZNPrE51Dz1kd4GVUkrVZsMG2L0bYmIa2IS0AU2YoCyA+PxzqyZCqaZCAwil7FRFAOE0Ttakrin9PLbXWDZP2cxzFz1HoG9g1dkUOTHGemJVMtfDXXdBdLR7iq2UUqp6JTUAl1/esAEsGtKECaBrV+jZ0+p7sXx5/Y+rlLtoAKGUnUqaMLn6QOw5uocL/nMBQ98cyo8HfixN1rNtzxqz+e2N31jTYw1Jj6axdCmEh8PUqW4rtVJuIyKRIvKRiBwXkd0ick016QJE5J8ikiYih0TkExHR0e9Vk1ASQIwa1bD9G9qECbQZk2qabA0g6nGjmC4ihSKSXW7pZGdZlPIKVw2EcQhz1s/h9NdPZ9muZbQObM2h3EN1ziZtXhq5Kbks+K9107nvPoiMdEuJlXK3V4ECIAaYALwuIr2rSHcXcBbQF4gFjgAve6qQSlVn71748UerGekFFzQwkwY2YYKKAYSrYlopr7O7BqKuNwqA/xpjQsstO20ui1Ke53SSEQxje23ihkU3kJWfxejuo9k0ZRMXdKrbnSdvdx5HVxwFfwdvbosiMhLuvtvN5VbKDUQkBBgLPGqMyTbGrAQ+Bq6rIvlpwJfGmDRjTB7wLlDd/UMpjymZ+6FkduiGaGgTJrAmDm3XDvbsgY0bG3Z8pezmW3uSuil3o+hjjMkGVopIyY1Cu36qFmHlkQ2MnQLpoRmE+YcxY+QMru93PSJ1f+qUNj8NgK2Rbcj5zZf777CaMCnVDHUDio0xKeXWJQPnVZH2DeAlESmpfZgAfFFdxiIyGZgMEBMTQ1JSkl1lbrDs7OwmUY6m4mQ5H++80xtoS6dOv5CUVPeJ4CrYZ71s276NbUnb6r17//7dWby4Pa+9toNx4/Y2rAxNzMny92GX5nY+bAsgqN+NAuByETkEHABeMca8XlUivUk0fXo+yuRvzyDPF84+GMYdl8yi3ZF2fPPNN/XLZJb18p/fYggIKGbAgO9JSmq+w2/o30dFLex8hAJHK607CoRVkTYF2AOkAsXARuD26jI2xszC9b8lISHBJCYm2lDcxklKSqIplKOpOBnOR1ERJCdb7++4ozvx8d0blM/W/2zlN36je6/utE9sX+/909KsuSC2betMYmLnBpWhqTkZ/j7s1NzOh50BRH1uFO9hXfjTgMHAAhE5YoyZXzmh3iSavpZ+Pn468BP92vXDIQ7IzmbVH/9OzzOH4Rgxrt55ZW/IZt2v68jz92VtQSS3TXYwevRQN5Tac1r630dlLex8ZAOV68/CgWNVpH0dCATaAMeBB7BqIHTedeU1q1dbs0B36wbx8Q3PpzFNmMDqe+FwWLNhZ2dDaGjDy6KUHezsA1HnG4Ux5mdjzH5jTLExZhXwEnCljWVRyu3yi/J5YOkDDJw1kJdXu/p6Op30zgCHo+aZqKuTNtdqvrS0MBrj4+Dee+0qrVJekQL4ikjXcuv6AZurSNsPmG2MOWSMycfqQH2miER5oJxKVWnJEuv14osbl0/r81vD5RDcPbhB+7dpY/WFKCyEZcsaVxal7GBnAFGfG0VlBqj/0ARKeUnyb8kk/CuBZ1c9i4iQlZ9lbag0jGt9GKchfX46AEtMNOPGNe6Jl1LeZow5DnwI/EVEQkRkKDAaeLuK5GuBiSISISJ+wBRgvzEm03MlVqqikgDioosal0+769vBvRBxVkSD8xgxwnpdvLhxZVHKDrY1YTLGHBeRkhvFJKA/1o3i7MppRWQ0sByro9wg4E7gEbvKopS7FDuLeXbVszy27DEKnYV0iezC21e8zZC4IVaCOsxEXZ0jy4+Qvy+fNAlgs4lg3gM2FrwZyMrKIj09ncKTfLrViIgItmzZ4vbjhISEEBcXh6Mhs17ZawrwJpAOHARuM8ZsFpFhwBfGmJLGGFOBGcA2wB/YBFzhhfIqBcDhw7BmDfj5QVNodThiBEyfDl98YQ3nWo+xOZSynZ19IKDuN4pxrnQBWGMT/N0YM8fmsihlq4zjGYx+dzTf7fsOgNsSbuPZC58lxD+kLFEjAoj0eVbtw1ITw4iRQt++jS5ys5GVlUVaWhodOnQgKCioXqNWNTfHjh0jLKyqrmH2cTqdpKamkpmZSbSXpy83xhwCxlSxfgVW37mSzwexRl5Sqkn43/+sS/q55za+z0FOSg5shcJ+hfi19mtQHgkJ1nxAv/4K27dbs1Qr5S22BhD1uFGMt/O4SnlC66DWFJtiYsNieWPUG4zoMuLERCUBRD2bMDnznaR/kAHAV8Tw7xbW9yE9PZ0OHToQHNyw9sGqIofDQUxMDLt37/Z6AKFUc2VX8yWAnQ/uhIVwJPoIbX/ftkF5+PhYZXn3XasZkwYQypu8XretVFO25+ge0rKtjs2+Dl/eu/I9Nt22qergAcr6QNSzBuLg5wcpPlLENkIJ7hnC8OGNKXXzU1hYSFBDZ2hSVfLz86OoqMjbxVCqWTIGvvzSem9HABHUJQi6gW/rxj231X4QqqnQAEKpKhhjePOnN+nzWh8mfTIJY6wh+E5tdSqtg1pXv2MDmzBFjozknR69mU08t9/eMtu2nszNlrxBz6dSDZeSYs38HBUFAwY0Pr/Oz3aGmdD6dzXcP+qgJJhZtgzy8hpfLqUaSgMIpSo5cOwAo94dxU0f38SxgmP4OfzIK6rjlbqBAcSGLT78a2tbNoVHMXFiPQuslFLKViW1Dxde2KAubW7Tvj307w+5udacEEp5SxP6b6GUdxljmL9xPn1e78OnKZ/SKrAVc6+Yy4I/LCDIr47NaxowjKsxhpdd00jceKNOEKSUUt5mZ/8HcE0k57QnL23GpJoCDSCUApzGydj3xnLNh9dwKPcQI7qMYNNtm5jQd0L9moI0oAbix5GbiJvzM23J409/qmfBlddlZGQwZcoU4uPjCQgIICYmhuHDh7N06VIA4uPjee6557xcSqVUXeXnl03WZlcAkXxxMgyHQ0sPNTqvkkntXJcYpbzC7mFclWqWHOKga2RXwgPCef6i57lpwE0Na0NezwCiIKOArCUHOccIGy7opqNqNENjx44lJyeHN954gy5dupCens4333zDwYMHvV00pVQDrFoFOTnQpw/ExtqUqatyWnwa3zfprLMgJAQ2boQDB6xmTUp5mtZAqBYr5WAKy3cvL/08PXE6P0/5mUlnTGp4B9R6DuPqE+nP/e0H8yQ9ueVejeebmyNHjrBixQqefvpphg8fzqmnnsqgQYOYOnUq48aNIzExkd27d3P//fcjIhX+rlatWsV5551HcHAwHTp04LbbbiMrK6t0e2JiIrfeeit33XUXrVu3pnXr1tx///04nTa1g1BKVcnu5kvgasKEPQFEQEDZxHZaC6G8RQMI1eIUFhfy9Mqn6ft6X8YvGM/RvKMABPkF0SG8Q+Myr+cwrkuWwA/7g9hzWnRptbRqPkJDQwkNDeXjjz8mr4ohUT788EPi4uJ47LHHOHDgAAcOHABg48aNXHTRRYwaNYrk5GQ+/PBD1q9fzx//+McK+8+bNw+n08l3333HzJkzmTVrFi+++KJHvptSLVVJAGHnNdk4rQDCrl9dJcFNSVmV8jR95KlalFV7VzHlsykkpyUDcGGnCzEY+w5QjyZMzgIn/5olgDBpUtMa6cPrvDUEqanf34Kvry+zZ8/m5ptvZtasWQwYMIChQ4dy1VVXMXjwYCIjI/Hx8SEsLIx27doB1kzUzz77LFdffTX33XdfaV6vv/46AwYMID09vXTyt/bt2zNjxgxEhB49epCSksILL7zAvfe2sJkGlfKQ9HT48UfrKf+wYTZmbGMTJigLIJYutW47ev9QnqZ/cqpFyDiewR8X/ZGhbw4lOS2Z+FbxfHntl8weM5tWga3sO1A9AojNU3cxeuFaznIc5MYb7SuC8qyxY8eyf/9+PvnkE0aOHMmqVasYMmQITz75ZLX7/PDDD8ydO7e0BiM0NJShQ4cCsGPHjtJ0Q4YMqdDs6ayzziI1NbVCUyellH2++sp6PfdcsHNuy5IaCLsCiO7d4ZRTrIBnwwZbslSqXjSAUCc9YwyXvnMpb61/C38ff6YNm8bmKZu5qLONDVxL1HEYV+M07H87jVPJ4YyzHNoJrjJjvLM0UGBgIBdeeCGPPfYYq1at4qabbmL69OkUFBRUmd7pdDJp0iTWr19fuiQnJ7Nt2zb69+/f4HIopRrHHc2XoKwPhF2/ukS0GZPyLm3CpE5axpjSjqvTE6fz0uqXeHnky3Rr0819B61jDcSRlUcJOJJPOgFc8pCNNSCqSejVqxdFRUXk5eXh7+9PcUlg6XLGGWewefNmunTpUmM+q1evLv07Bvj++++JjY0lPDzcbWVXqqUyxj0dqAHbmzCBVcY33rDK/MADtmWrVJ1oDYQ66fyW/Rs3f3wzdy++u3TdJV0vYfGExe4NHqDOAcRPz6YBsCYsmotHeqm9v2q0gwcPcv755zN37lw2bNjAr7/+yvvvv88zzzzD8OHDCQ8PJz4+nhUrVpCamkpmZiYADz74IGvWrOHWW2/lp59+Yvv27Xz66afccsstFfLfv38/d999N7/88gsffPABzz77LPfcc483vqpSJ71Nm8qGRe3Tx9687W7CBDB8uFUTsWKFNeysUp6kNRDqpJFbmMs/vv8HT618iuyCbAJ8Anh42MO0C7U6rzZ4aNb6qEMTJmeBk4IlGQQCMRNi6jNptWpiQkNDGTJkCC+99BLbt28nPz+fDh06cM011zBt2jQA/vKXv3DLLbfQuXNn8vPzycrKom/fvixfvpxp06Zx3nnnUVxcTKdOnbjiiisq5D9hwgSKi4sZPHgwIsJNN92kAYRSblK+9sHu24XdTZgA2rSBhARYuxaWLy+boVopT9AAQjV7TuPk3U3v8tBXD7E3ay8Ao7qP4pkLnikNHjxXmNprIHa9d4jAgiJ2EsLVD4d6qGDKHQICAnjyySdr7DA9ZMgQkpOTSz8fO3YMgISEBBYvXlxj/r6+vrzyyiu88sor9hRYKVWtL7+0Xm1vvgRuacIEVlnXrrWCHw0glCdpAKGatbyiPM5961zW7l8LQP92/Xn+ouc5/7TzvVOgOgQQG15IoxWwt1s0HTt6plhKKaWql5trPcUHuPBC+/N3RxMmsAKIJ57QjtTK8zSAUM1aoG8gnVp3Yl/WPp44/wkm9puIj8OLbYJqCSCKsooIWX8QgF53xniqVEoppWqwYgXk58MZZ0DbtvbnH3J6CLmSiyPI3q6nQ4ZAaChs3gypqdChkXOhKlVX2olaNSvrf1vP5fMv5387/1e6bsbIGWy/czs3DrjRu8ED1NoHYuNrmfgZJ5t8Irjsj4EeLJhqbpKSkrTpklIe4tbmS0CfD/rAPyGwo73XfX9/+N3vrPdLl9qatVI10gBCNQur963m8vmXM2DmAD5N+ZQnVjxRui06JJpgv2Avlq6cWmogds6yRl/KHhxj6yRFSimlGs5d8z94gs4HobxBmzCpJu2bXd/wtxV/46ud1vSgQb5B3DLwFh4e9rCXS1aNGgKI3NR8Wv96mEKEIQ+6oY5cKaVUve3fbw3hGhICZ53lnmM4i5zgpMK8LnYpCSCWLrVuQbWMIq6ULTSAUE3WnPVzuGHRDQCE+ody+6Dbueese4gOifZuwWpSQxOm1U9n4AB+CmrDPZf5ebZcSimlqlTy5D4xEQIC3HOM70/5Hn6Dgn0FBHSw9yBdu8Kpp8Lu3bB+vdWPQyl30zhVNRnH8o+xNnVt6edR3UcRFx7H9POms/vu3Tx1wVNNO3iAGmsg5h9rz3R6UXzlKfqESCmlmgiPNF8S1+KGbnoi2oxJeZ7+jFFet+3gNqYumUrcP+IY9e4o8ovyAWgd1Jpf7/qVPyf+mcigSC+Xso6qCSByc+Hdj3z4hmgufzjCCwVTyntEJFJEPhKR4yKyW0SuqSHtGSKyXESyRSRNRO7yZFlVy+J0lnU+dlcHaoCz958NX0NAO/dUcWgAoTxNmzAprygoLmDR1kXM/GEm//u1bESlfjH9SDueRscIa4IEX0cz+xOtJoD45GNDVpYwcCD07OmFcinlXa8CBUAM0B/4TESSjTGbyycSkShgMXAP8AHgD8R5uKyqBfnpJ8jMhI4doVs3b5em4c4/37rtrFwJx49b/TmUcqdm9utMnQx2H9nN4H8PJu24NSJRkG8Q4/qMY8qgKSTEJni5dI1URR8IYwxMXsfDhBI7tgug/R9U3UyfPp0PPviATZs2ebsoDSYiIcBYoI8xJhtYKSIfA9cBD1VKfi/wpTFmnutzPrDFY4VVLU755ks29232qMhIGDQIVq+Gb76BSy7xdonUyU4DCOV2GcczWL57OWN7jQWgY0RHWgW2Iio4ilsG3sJ1/a6jVWArL5fSJlXUQOxLOk501nHOoJBzrtf/cieTG264gczMTD799FO35D916lTuuOMOjx3PTboBxcaYlHLrkoHzqkg7BNgoIquALsBq4E/GmD2VE4rIZGAyQExMDElJSXaXu96ys7ObRDmaiuZwPt57rx/Qmg4dNpOUlOG+A/0JivOKSXo1Cdw0BVC3bvGsXh3Pm2/uIzh4u3sOYqPm8PfhSc3tfOivGeUWR/OO8mnKp/x383/5YvsXFDuL2XX3LjpGdERE+OaGb4gOibZ9ODuvqyKAWLQ5lKc5k1EJeVwZe5J9X+VWoaGhhIaGersYjRUKHK207igQVkXaOOAM4EJgI/AMMB8YWjmhMWYWMAsgISHBJCYm2lfiBkpKSqIplKOpaOrnIzvbmsHZ4YA77+xN69buO9byX5dDLgw7dxg+we6Z8NTXF95+G37+OY7ExKbf8q+p/314WnM7H9qJWtkmpzCHOevncPn8y4l+LpprP7qWT1I+wRjDyK4jOZZ/rDRtTGjMyRc8QJVNmN5+G1IJZti9zaQjuLLFnj17uOKKKwgLCyMsLIzf//737Nu3r0Kap556ipiYGEJDQ5k4cSKPP/448fHxpdunT59Onz59St/PmTOHzz77DBFBRJrL06psILzSunDgWBVpc4GPjDFrjTF5wOPA2SKiIw8o2yUlQWEhnHkmbg0eAEyxsd648VfX4MEQFgZbtsDeve47jlKgNRCqEYwxpf0YAAqLC5n86WQKigsQhPNOPY+xPcfyh95/ICY0xosl9aBKNRBbk4tZs8aHsDAYPdqL5VIeZYxhzJgxBAYG8vXXXyMi3H777YwZM4avv/4agHfffZfHH3+cV155hXPPPZcFCxbw9NNP07qaXzJTp05ly5YtHDp0iLfffhuAyMhmEZSmAL4i0tUYs821rh+wuYq0GwBT7nPJ+5PwaYPyti+/tF7dOfpSiZIAQnzc96fs52d1pl60yBpZ6o9/dNuhlNIAQtXP0byjrNizgiU7lvDZts84lHuID878AICIwAgeHPogHcI6MKbHmJYTNJRXKYBYP+EX3uA4W4d1IzhYH6LWhzxe/Y125mUzmTxwMgCzfpjFLZ/eUm1a8+ey36MDZw3kxwM/1pqusb766iuSk5PZsWNHaY3CO++8Q5cuXUhKSmLUqFG89NJL3HDDDUyaNAmAhx9+mGXLlpGSklJlnqGhoQQFBREQEEC7du1sK6u7GWOOi8iHwF9EZBLWKEyjgbOrSP4WsEBEZmAFGI8CK40xRzxWYNViLF5svbp1/ocSrluDOwMIsIKhRYuszuEaQCh30gBC1WrXkV38c90/+frXr/nhwA84jbN0W5ugNuzP3V/6+S+/+4s3ith0lDRhcjgozCqi9c+ZtMNJ/DX+3i2X8qgtW7YQGxtboTlSp06diI2NZevWrYwaNYqtW7dy8803V9hv8ODB1QYQzdwU4E0gHTgI3GaM2Swiw4AvjDGhAMaYr0XkEeAzIBhYCVQ7Z4RSDbV9u7W0bm01/XEnY0xpXZo43B9AgFUDUVxcoTWtUrbSAEJVkJqVyurU1fg6fBnVfRRgzRD992//DljzMgyJG8L58edzSddLOLPDmaxYvsKbRW5aSmogfHxY9WwmAcZJin84k8YHebdczVBdawQmD5xcWhtRmx8m/9CYItWZMabaPj7l15+U/YCqYIw5BIypYv0KrE7W5de9DrzuoaKpFuqLL6zXiy5y/49sT/R/KNG5M5x2Gvz6qzXHRUIzHxldNV0aQLRgWflZJP+WzOrU1Xy/73tWp65mX5bVyXNI3JDSAKJ3dG8ePfdRzj7lbM7peA6h/s1+VBj3KdeEKXV2OrFA4XkxleeVUye5Xr16kZqayq5du0prIXbu3Mn+/fvp0aMHAD169GDNmjXceOONpfutWbOmxnz9/f0pLqnlUko1WEkAMXKkBw5WUmnvgfuAiBUUzZxpNWPSAEK5iwYQLYDTONl5eCcb0jYwJG4IsWGxADz69aPMWDOjQtrwgHDO7HAm53Y8t3SdQxzaNKmuXAFE1kF/YvYdogjh3MfaerlQyp2ysrJYv359hXVdunShX79+TJgwgRkzZmCM4Y477uCMM87gvPOs6Q/uuusubrzxRgYNGsSwYcP46KOPWL16dbWdqAHi4+P54osv+OWXX2jTpg0RERH4+enEhErVR24uLFtmvfdE/wdP1kBAxQDikUc8c0zV8mgAcZIpchbx/ub3+eXgL/xy8Be2Zm4l5WAKOYU5AMwZM4eJ/SYCMKjDIM5ofwaDYgcxuMNghsQNoXtUdxyij8sbzPV0ePnC9oTiZHN4JBeco/0fTmYrVqxgwIABFdaNHTuWhQsXcuedd5aO633BBRfw8ssvlzZbGjduHDt37uShhx4iJyeH3//+99x6660sWrSo2mPdfPPNJCUlkZCQQHZ2NsuWLWtW44Yr1RQsXw55eTBgAHhiPAJPBxDnn2+N4/Htt3DsmDW0q1J20wCiGSkoLmDP0T3sOrKrwuIQB/+54j+AVVtw08c3kVuUW2Hf2LBY+sb0pU1Qm9J11/a9lmv7XuvR73DSc9VAHFvnRyj5BI+O9nKBlDvNnj2b2bNnV7t94cKFJ6w7dqxs+oNHHnmER8o9Irziiivo0qVL6efp06czffr00s9t27ZlyZIljSu0Ui2cR5svgUebMAG0agVnnWUFEF99BVdc4ZnjqpZFA4gmwGmcZOZksv/Yfg4cO2C9Zh9gTI8x9Im2JpH6+8q/8/D/HsZwYsfSMP+w0k6bDnFwa8KtBPgE0COqB92jutO9TXdaB7l5lhxlcTpJpTftj+eTgw8XT4/ydolUE5WTk8Prr7/OiBEj8PX1ZcGCBSxatIgFCxZ4u2hKndQ8HUB4ugYC4NJLrQDis880gFDuoQGEGxzMOchv2b9xNP8oB3MOcjD3IJk5mWTmZBIZFMkDQx8AIL8on7h/xHEo91CFoVFLtA9tXxpARAVHISJ0DO9IfKv4E5byXrj4Bbd/R1WN4mK+5TaigZ2xUVzSScfQU1UTEb744guefPJJcnNz6dq1K2+//TZX6N1eKbfZuRNSUqyn9EOGeOigTsDHtXjIpZda/R8++8yqGNeBPJTdbA0gRCQSeAO4CMgEHjbGvFNFOgGeBia5Vr0BPGiMsW8mpzpKy07jaP5RjhccJ6cwp8ISFx7HsFOHlaZ7dtWzZOVnsX3vdp5JfYaj+UfJys/iaN5RPrr6IwbGDgTgsWWP8dq616o8Xq+2vUoDiADfAI4XHMdpnEQGRRIbFkv70Palr6fHnF6637V9r2Viv4n4+WiHyaasMM+PAHoD0O7aFjiRnqqzoKAgvvrqK28XQ6kWpaT24cILwddDj1D92viRWJRIUlKSZw4InH46xMXBvn3WcK4DB3rs0KqFsPu/z6tAARCDNdvoZyKSbIzZXCndZKwxwfthTa+yFNgJ/LOmzFOPpXL757dTUFxAfnE+BcUF1vuifK7vdz1X9b4KgC+3f8nUpVNLt1VOn3F/BqeDfBcAACAASURBVOEB4QBc/cHVfLP7myqPN67PuNIA4ljBMZ7/7vmyjRkV0x7KPVT6/pSIU+gR1YPwgHCigqNoE9SGqOAoooKj6BjRscJ+u+/eTavAVrUGBgG+ATVuV95njOHHHy4lAtgqYVz7f9psTCmlmhKP93/wEhGrFmLmTKsWQgMIZTfbAggRCQHGAn2MMdnAShH5GLgOeKhS8uuB540x+1z7Pg/cTC0BRODOQBKvTqxyW3hAOKv8VwHgV+THn3P/TEZ4BlMmTylN89arbxGaG0rODTmEx1kBxMR/T+TujXcjIghS4dXfx59Vd1h5GgxfFnyJQxx8+/dvGTRsEOEB4TjudFC8upjuw7pDZ+s441eM59yXz6U6q1h1wrpe83vR6rxWAOx+ajepL6fS8aGOxN0ZB8DBzw7yy82/1HR6TlDV/pGXRNLj39Y49Hl78/hx8I/1yrPK/SOALWVp1vReQ9HhojrnGRAXwMA1ZVe3kv0HbRqEX6QVWG0et5mjy4/Wq6xV7V/Vea6Pmv6dio4WkXE0Fgc+bBocSWh4y5gkTCmlmoO8vLLhW0eM8G5ZPKF8APHYY94ujTrZ2FkD0Q0oNsaklFuXDJxXRdrerm3l0/WuKlMRmYxVY0EX6UJUdjWdUrOhgAIAHDiIIopWwa2YP3g+vuKLn8OP8OfDkWxhy7otbN2+FYBO+Z0gq/ovVZIngD/WcJwDAwcS/P/t3Xd4VFX6wPHvSe/0hNA7CkgTRaoBsaOgYkNRBEFhsQuyxZWfurrqqgssFgRRQEFWAUFBYZUgIAihCkjoNSS0kBDSM+f3x5lMMmGSTMIkM5O8n+e5z8zcOefekzs3d+a9p50IIZdcOAEkwu7Nu9nts9sk3AmcLH6bjmzbuM021X1+/v079rM/dr9ZF1f2bTrKn7g3kcTYRLMusezbdJTfkmexr5o9TonHtKhsS7bD/OvWrDPBCcCBspfVUX5Hx7ksSvqctIbnaIs/NXm2w3JiY4+UbeNVVFpaWqlV9zVq1LAbnagqy8vLq7S/NTMzs1KbTQjhyX7+GdLTzfCt0dGVt9+shCx23LoDgoENlbff/v0hMBA2boSkJIiSVrXChVwZQIQBRW8RpwCORiAumjYFCFNKqaL9ILTW04HpAFd3ulr3WN7D6QIpX0VAVMEY/FnxWaAhoH4AysfcHc75IQdL1qUdmEuyfs9629jr+fn9avvhG2R6SOV2zSXvnbLNFusov2+4L37h5iPK655H7ijn7+oDDvP7BPnY7spbci3k9M0p0zYd5V+/Yb3dWPRZe7JwMFhUsS75nPa45nNylL8iP6d162B/NjTgBCPvroWvjM8PQGxsbKlzFfzxxx+EV5PByi9cuFBpf2tQUNAlc1QIUV0tWWIeBw2q3P1asixc3HERKmHOicJCQ6FfP/jhB9N0a/jwyt2/qNpcGUCkARFF1kUAjm61FU0bAaSV1ola+SsCG5S/L0Bg9KV5/euUo1NyoToWR/n9Ivzwiyj/oXWU3zfYF9/g8g/h4Ci/j5/PZR1PW/7a9usdHeeycNnnVEp+V39Oc+aYx4f4Al//ruXerhBCCNeyWGDpUvP8zjsrd9+BDQLptq0bcVviKnfHwMCBJoD4/nsJIIRruXJgr72An1KqdaF1nYCiHaixruvkRDohvEJWFixYYJ4PY46MmSeEEB5kyxZISDAjE3XuXLn79gn0IaxTGDSv3P2C6QcBsGIFZGeXnFaIsnDZrxyt9UVgIfCqUipUKdULGATMcZB8NvC8UqqhUqoB8ALwmavKIkRl+/57SE6GTqH7uIqd4CvzPwjXeeCBBxgyZIi7iyGE18pvvnTnnWaEouqiWTNo1w5SU2HtWneXRlQlrr5NOhbTTegUMA8Yo7XepZTqo5RKK5TuY2Ap8DumK+r31nVCeKX85kvD6v1onkgNRJWnlCpxGS7tBYTwGIUDiMqWeTyTPSP2wKzK3zcU1ELkN+ESwhVcOg+E1vocZn6HouvXYDpO57/WwATrIoRXO3vW1ED4+MDQuivgMBJAVAMnTxYM4fXdd98xatQou3XBwcHuKJYQoogjR2D7dggLA3eMbZF7NpfEWYnQovL3DabT+DvvwKJF8N571asGRlQc+ZUjxGVasABycmDAAIj2PWVWShOmKq9+/fq2pWbNmpesq1HDjCH8/PPP07p1a4KDg2nevDmvvvoq2YUaI0+cOJFu3boxe/ZsmjdvTkREBEOGDCE5OfmSfb7zzjtER0dTp04dRo0aRVZWVuX8sUJ4sfw777fcYoY1rWzaYh0fxk2/uHr0gPr1TSC1bZt7yiCqHgkghLhMtuZLwzBDfYDUQAibGjVqMHv2bP744w+mTJnC3Llzeeedd+zS7N27l6VLl7J06VKWLVvG+vXrmTRpkl2alStXcvjwYVatWsWcOXOYP38+H3zwQSX+JUJ4J3cN35pP51kDCDfd+ffxgcHWtiELF7qnDKLqkV85QlyG/fth/XoICbFeoCWAcAml3LNUhFdeeYUePXrQrFkz7rjjDp555hnmzZtnl0ZrzaxZs+jQoQO9e/dmxIgR/PTTT3Zp6taty9SpU7niiiu47bbbGDx48CVphBD2UlIgNtZUCt92m5sKkT/dkBsrpu+6yzwuWuS+MoiqxaV9IISobubONY93323a15Jn/aaQAEJYzZs3j6lTp3Lw4EHS0tLIzc0lICDALk2LFi0IC7N1E6NBgwacOnXKLk2HDh3wKXReNWjQgPj4+IotvBBe7ocfTBPT66+H2rVLT18R3N2ECUzfj5o1YdcuiI+Htm3dVxZRNcivHCHKSeuCAGLYMOvK/BoI6QNxWbR2z+Jqq1evZtiwYdx555189913bN26lZdeesmuDwSAv7/9RIdKKSwWS5nTCCHsffONeXRX8yVwfxMmgIAAM6kcSC2EcA0JIIQop19/hQMHIDoabrjBulKaMIlC1q5dS8uWLW0dpVu3bs3Ro0fdXSwhqoWLF+G778xzt06j4gFNmMDUlIMEEMI15FeOEOX0+efmcdiwQhUO0oRJFNKmTRsOHTrEggULOHDgAFOmTOHbb791d7EqnVKqtlJqkVLqolLqiFJqaCnpA5RSe5RSxyurjKLq+f57yMgwoxA1buy+cnhCEyaAm2+G4GDYuBGOHXNvWYT3k185QpRDRgZ89ZV5/uijhd6QJkyikCFDhvDUU08xduxYOnfuzNq1a5k4caK7i+UO04BsIAp4CPhQKdW+hPTjMROSClFuCxaYx/vuc285PKEJE5jBPm65xTxfvNi9ZRHeTwIIIcrh228hNRW6dYN27Qq9IU2YqqUhQ4agHXSiUErx7rvvcubMGS5cuMCCBQsYM2YMmZmZtjT//Oc/iYuLs8v35JNPcubMGdvr+fPn8/XXX9ulcZTPEymlQoF7gJe11mla67XAEmBYMembAw8Db1ZeKUVVk5YGy5aZ525tvgQe04QJpBmTcB0ZhUmIcshvvmRX+wASQAhxqTZAntZ6b6F124Hri0k/FfgLkFHSRpVSo4HRAFFRUcTGxl5+SS9TWlqaR5TDU7jzePz8cz0yMtrTvn0K+/dvZf9+txTDsE7elmvJdfv5UbOmH35+PVm9WrFw4Xpq184uPVMFkf8Xe952PCSAEKKMEhJgxQrw94cHHyzyZn4fCGnCJES+MCClyLoUILxoQqXUXYCf1nqRUiqmpI1qracD0wG6deumY2JKTF4pYmNj8YRyeAp3Ho+pU83jqFE13P6ZnEk7w0524ufv5/aygJkPY8kSOHasp61Gwh3k/8Wetx0PuU0qRBl98YWpaBg4EOrUKfKm1EAIUVQaEFFkXQRwofAKa1Ont4GnKqlcooq6cMGDmi8BNfvW5OqtV3vMmT3UOoTBl1+6txzCu8mvHCHKQOsSmi+BBBBCXGov4KeUal1oXSdgV5F0rYFmwBqlVCKwEIhWSiUqpZpVQjlFFfHdd5CZCb16QcOG7i4N+EX4Ed45HDygLAB33GEmPt24Efc27RJeTX7lCFEGW7aYmTzr1oVbb3WQQIZxFcKO1voiJhh4VSkVqpTqBQwC5hRJuhNoDHS2Lo8DSdbnMuikcNp//2se3T36kqcKCYG77jLPpRZClJf8yhGiDPJrH4YONTN7XkKGcRXCkbFAMGZo1nnAGK31LqVUH6VUGoDWOldrnZi/AOcAi/V1XvGbFqJAcrKZ/0EpuOced5fGSP0tlT0j9pixxzzEQw+Zxy+/NDXrQpSVdKIWwknZ2QV3axw2XwJpwiSEA1rrc8BgB+vXYDpZO8oTCzSq2JKJqmb+fHOtvvFGz2i+BJBxIIPEWYnQ390lKXDDDRAZCfHxsHUrdO3q7hIJbyO/coRw0rJlcPYsdOgAXboUk0gCCCGEcJvPPjOPw4e7sxT2IrpH0HZGW7jd3SUp4OcH999vnn/xhXvLIryT/MoRwkmFO0+r4mYUlWFchRDCLXbvNh2DIyJg8CX1Xe4T3DKY6JHR4GF3+fNHY5o/v+CrSwhnSQAhhBNOnzbtan18CtqOOiQ1EEII4Rb5N3nuv990FBYl694dWrQwcxt50fxlwkPIrxwhnDB7NuTkmJGXoqNLSCgBRLUyfPhwlFIopfDz86NJkyaMGTOG5ORkp7cRGxuLUoozZ84Uu4+BAweWOZ8Q1UluLsyxjutVbB81N7m46yInpp2wzUjtKZSChx82z2fOdG9ZhPeRXzlClEJr+OQT83z06FISyzCu1c6AAQM4efIkhw8fZsaMGSxdupSxY8e6u1hCVCsrV8LJk9CqFfTs6e7S2EtZm8K+cfvgJ3eX5FIjR5pA4ptvQO5FiLKQXzlClGLNGjNSRYMGcNttpSSWYVyrncDAQOrXr0+jRo246aabuP/++1mxYoXt/ZSUFEaPHk1kZCTh4eHceuutxMXFubHEQlQ9+c2Xhg8voY+am+g86zipHviLq0kTU7OenW1q2oVwlgeezkJ4lunTzeNjj5mRK0okTZiqtYMHD/LDDz/g7+8PgNaa22+/nRMnTvDdd9+xdetWevbsSf/+/Tl58qSbSytE1ZCcDIsXm8Bh2DB3l+ZSnhxAQEHN+vTpMieEcJ7MAyFECc6dg6+/Ns9HjnQigwQQLhOrYsuUPqxrGN02d7skf4yOsa2LuzqOtC1pDvMXTlcWP/zwA2FhYeTl5ZGZmQnAe++9B8CqVavYtm0bp0+fJjg4GICXX36ZFStWMGfOHCZMmFCufQohCsyZA1lZZm6DJk3cXRoHrF8LnhpA3H676dsXH29q3Pv2dXeJhDfw0NNZCM8wd675YrrpJmje3IkMMoxrtdO3b1+2bdvGxo0beeqpp7jtttt4+umnAdi8eTPp6enUq1ePsLAwwsLCiI6OZufOnRw4cMDNJRfC+1ks8J//mOdjxri3LMXx9BoIP7+CG2T5Ne5ClEZqIIQohtYFF9NRo5zMkM/TGuF6ofLWCJSUv3ANhauEhITQqlUrAKZMmUK/fv147bXXmDRpEhaLhaioKNasWWNLn5aWRlhYGBEREU5tPyIiwmGwcf78eXx8fAgPD3fNHyKEF1qxAvbtg8aNYdAgd5fGMU8PIMAEEP/4h6lxnzwZ6tRxd4mEp/Pg01kI99qwAXbtgnr14M47nchgbb6kpflStfbKK6/w1ltvkZCQQNeuXUlKSsLHx4dWrVrRqlUrWrZsSatWrYiMjHRqe23btmX37t1kZGTYrd+yZQtNmzYlMDCwIv4MIbzC1KnmcexYJ/qouYuHN2ECaNYMbr7Z1LjnD4crREk8+HQWwr0++sg8Dh8OAQFOZLA2X9JS+1CtxcTE0L59e15//XUGDBhAr169GDRoEMuXL+fQoUP89ttvvPLKK3a1EgA7d+5k27ZtdovFYuHhhx/Gz8+PRx55hM2bN7N//35mzZrFv//9b8aPH++mv1II99u/H5Yvh8BAePxxd5emeN5QAwEFnak/+KCgO58QxfHw01kI9zh1CubPNy2RnnzSyUzSgVpYPf/888ycOZOjR4+ybNky+vfvz6hRo2jbti3Dhw8nPj6eBg0a2OXp168fXbp0sVvS09OpUaMGa9asIS8vjzvvvJPOnTszefJk3nvvPZ50+uQUouqZNs20HB06FOrWdXdpiuctAcQdd0DTpqZJ2JIl7i6N8HSeWuEnhFt98okZF3vgQGjRwslM0oSp2vnss88crh86dChDhw61vZ48eTKTJ08G4MKFC3b9FmJiYtCljJ3Ypk0bFi5cePkFFqKKSEuDTz81z596yr1lKVX+3XwPr5z284PnnoNnn4V//QsGD3Z3iYQnk186QhSRkwMffmiel+mLKX8EJmnCJIQQFWrOHEhNhV69oEsXd5emZLYaCC8YnG/kSKhVC9atg/Xr3V0a4ckkgBCiiMWL4cQJaNsWBgwoQ8b8GggZwlUIISpMXh78+9/mucfXPgDa4h1NmADCwgqGw33nHfeWRXg2Lzidhahc+aN6jBtXxu4M+QGE1EAIIUSFWbQI9u41Iwfdc4+7S1O6enfVo+2MtnCdu0vinHHjzMAhixeb4yyEI9IHQohCtm83M3GGh8Ojj5Yxs3SiFkKICqU1vPmmeT5+vPuHbtVao6w3jc5lnOPvq/5OYloiZ9LPcDbjLCmZKeRacsm15PJ086eJIQaA+Tvn8/Hmj2kQ3oCG4Q1pFNGI1rVb065eOxrXaIyPct/3SHQ0DBsGM2fC++8XNOkVojAJIIQoJH9G0+HDTRBRJjKM62Up/EUsLl9pHbOF8EYrV8KWLRAZCY89Vrn7tmgLu0/vZs2RNWxK2MTWxK00q9mMRfcvAiDQN5Bpm6YVmz8zL9P2/FDyIWIPxzpMVzu4NqfHn7YFEacunqJeSL1KvT6+8IIJID77DCZNgqioStu18BISQAhhdeoUzJ1rnv/pT+XYgNRAlJu/vz8ZGRmEhIS4uyhVRk5ODn7uvj0rhAtpbX7MAjz/PAQHV85+v9/7PTO2zuCXI79wLuOc3XunLp6yPQ8NCOWD2z6gdnBt6obUpU5IHWoG1SR3Qy45f+RwPPW4Le0jnR7hmobXcCL1BAkXEjiacpT4s/HsPr2byNBIW/CgtebKaVcSHhDOgBYDGNBiADe2uJE6IRU7VfSVV5oJVJcsMX0h/vWvCt2d8ELy7SKE1dSpkJlphm5t27YcG5BhXMstMjKSEydO0LBhQ4KDg6Um4jJZLBaSkpKoUaOGu4sihMv8+KMZGahu3XLe5HHSzlM7qRVUi4YRDQGIPxvP4j2LAWgY3pC+TfvSo1EPukZ3pWNUR7u8Y64Zc8n29i3bR9LUJPz/5G9b1zCioW37RaXnpNueJ6YlAnAk5Qgzt85k5taZ+CpfYprFcPeVd3Nf+/uoG1Ixk2BMmmQCiGnTTI1EdHSF7EZ4KQkghMCMKT7NWvP80kvl3IgM41puERERACQkJJCTk+Pm0lSszMxMgoKCKnw/oaGh1PXk2bWEKAOt4e9/N89fesmMFuRK8Wfi+WrXV3y16yt2n97N3/r8jdf6vwbA3VfebasBaFazWZlvcNToXQOdq0lomeBU+hD/gprY6PBoTo8/zbbEbfzv4P9YcWAFq4+s5qdDP/HToZ/oXL+zLYCwaItL+0506QJ33WU6rf/zn2CdykYIwIUBhFKqNjATuAk4A/xZa/1lMWknAX8Fsgqt7qi1Puiq8ghRFp98AsnJ0LMn9O5dzo3IMK6XJSIiwhZIVGWxsbF08fSB64XwMIsWwaZNpi3+2LGu2WZqVirzfp/HjK0ziEuIs62vHVwbP5+Cn0fNajZj1NWjyr2fyPsiibwvkoRY5wKIonyUD12ju9I1uisTek0gOSOZpXuX8tOhn7iuUcHQToPnD8bXx5fHOj/Gra1uxd/Xv4StOmfSJHPsP/rITDLXrNllb1JUEa6sgZgGZANRQGfge6XUdq31rmLSf6W1ftiF+xeiXLKz4b33zPNy1z6ADOMqRDGcvcGklBoPPAo0tab7QGsto9FXczk5MHGief7KK+CqrlLjlo1jzo45AEQERnDXFXdxf/v7GdBigEt+fFeUWsG1eKTTIzzS6RHbuuSMZH488CPZedks3rOYeiH1GNZxGGOuGUOr2q3Kva+OHeGhh+CLL+CvfzWPQoCL5oFQSoUC9wAva63TtNZrgSXAMFdsX4iKNG8eHD8O7dqZ/g/lJp2ohShO4RtMDwEfKqXaO0ingEeAWsAtwDil1AOVVkrhkaZPh337oE0bePzx8m0jKzeLT7d+yq/HfrWte6zzY8Q0i2HuXXNJfCGRzwZ/xq2tXXPnvrCMAxmk/pYK5126WTu1gmtx+JnDvD3gba6seyWn00/z3ob3aDO1Dbd/eTt7z5Z/QofXXzfzQnz5JWze7MJCC6/mqhqINkCe1rrwGboduL6EPHcopc4BJ4H/aK0djjSslBoNjAaIiooiNjbWNSW+DGlpaR5RDk/hzcfDYoFXXrkGCOWOO/7gl1+Syr2t4GPH6A5YtPba41ERvPn8qAjV7XgUusHUQWudBqxVSuXfYJpYOK3W+u1CL+OVUt8CvYD5lVVe4VnOnSsYeemtt8C/jL/tz6af5aO4j5i6cSpJF5O4ueXN/PDwDwD0a96Pfs37ubbADhx5/QiJnyXCeGBwxe0nOjya8b3G82LPF9mUsIkP4z5k3u/zWHlgJRGBBc1D8yx5+Po439S2WTN4+mkzEtNzz8Hq1dLVT4ByxVjhSqk+wH+11vULrRsFPKS1jnGQvh0mFk8CugPfAM9rreeVtJ9u3brpuLi4kpJUitjYWGJiYtxdDI/hzcfj229h8GBo1AgOHDB3Wcptzx648krSGzcm5OhRl5XR23nz+VERPOV4KKU2a627VcJ+ugC/aq2DC617Ebhea31HCfkUsAX4WGv9kYP3C99cunr+fPfHGGlpaYS5unevF3PF8Zg8uTWLFzekc+dk3ntvu9M/XE9knODr41/zQ+IPZFrM/AstQ1tyf+P7GRA5oHJHensTWAGZz2YSNKjiB1AoLCUnhZ0pO+lVtxcAeTqPkXEj6VSzE/c2vJdGIY2c2k5amh/Dhl3L+fMB/PWvuxkw4FTpmUrdpvy/FOYpx6Nfv35OfTc4VQOhlIql+NqEdcBTQNHejxHABUcZtNa7C738VSk1GRgClBhACOFKpvbBPH/xxcsMHvI3iAzjKkQRYUBKkXUpQGlTNU7CNLOd5ehNrfV0YDqYm0ueEJR5SnDoKS73eGzbZoYR9fWFOXNq0aGDc9uav3M+w74ZhsbcIL2l1S280OMFbmh+g1uGiN49YzenOEVQcJBbzo9BDLI9X3t0LUd+OcKR9CMsTVjKXVfexfie4+06Yxfn3Xdh5EiYNasdL73UruyTrRYh/y/2vO14OPVLR2sdo7VWxSy9gb2An1KqdaFsnYDiOlBfsgtM21chKs0338D27dCwITzxhAs2KMO4CuFIGmW4wQSglBqH6Qtxu9Y6q7h0ourKyzPXZYsFxo2DDh1KSGvJ48C5A7bX/Zv3JyIwgsc6P8bvY35n+UPLGdCikmsd7ApoffSAe0u9m/Rm19hdjOwyEn9ffxb+sZAeM3vQZ1YflsQvwaItxeYdPhyuvRYSEuBvf6u8MgvP5JLTWWt9EVgIvKqUClVK9QIGAXMcpVdKDVJK1VLGtcDTwLeuKIsQzsjLKxhT/OWXwSXD8ksNhBCOlOkGk1JqBKZvxA1a6+OO0oiqb+pU2LjR3OB59VXHaS5mX+Q/G/9Dm/+0of/s/uRacgGIDI3kxPMn+HTQp3SILCHyqCTaYm0q7iFfDe3qtWPGnTM4/Mxh/tz7z9QMqsnao2t5evnT5Fnyis3n42OGc/X1NZ/Phg2VWGjhcVx5Oo8FgoFTmKZIY/KHcFVK9VFKpRVK+wCwH3MHajbwltb6cxeWRYgSffGF6bLQvDk89piLNioBhBCXKMsNJqXUQ8AbwI0yL1D1dfCgGTIU4MMPoej0MCcvnOSvP/2Vxu835qnlT3Ew+SC+ypfD5w/b0oQGhFZegUuh8zwrgMgXHR7NGze8wdFnj/L+ze/zWr/XbCNQnbp4ijfWvMG5jHN2ebp0MU1+tTbNmTIz3VFy4QlcNg+E1vocxYwvoLVeg2kHm//6QVftV4iyyskpGNVj0iQX9H3IJ02YhCjOWOBTzA2ms1hvMFkH4Fiutc7/fngdqANsKtTcZK7W+snKLrBwj7w8eOQRSE+H+++HOwp1sz+feZ7nf3yeL37/guy8bACua3QdL/R4gbuuuKtMIwtVKg9qwuRIeGA4z173rN26aRun8eovr/KPNf9gZJeRPHfdczSv1RwwfQcXLoTdu00N/jsyU0u15KGnsxAVZ9YsOHQIrrjCTJDjMlIDIYRDWutzWuvBWutQrXWT/EnktNZrCgUPaK2ba639tdZhhRYJHqqRt9+GdesgOhqmTbN/LzwgnNjDseTk5XDXFXexbsQ61o9cz5B2Qzw3eMDzmjA5o3/z/tzU8ibSc9KZunEqraa24v6v72fTiU0EB8OcOaYp07vvQjUalVoU4kWnsxCX78KFgtqH//s/cwF0GZlITgghym39+oK+aZ/MyGX5ibl0n9GdkxdOAuDr48ungz5l71N7WXj/Qno27unG0jrPU5swleT6Ztfz48M/sv3J7TzS6RF8lA8Ldi3g2hnX8tLKl+je3TQz0xoefhhOn3Z3iUVl86LTWYjL9+abcPKkGUliyBAXb1xqIIQQolzOnIH77oPcXOh93yaejG/OsEXD2HhiI59s+cSWLqZZDK1qt3JjScvBw5swlaRjVEc+H/w5h545xISeE4gIjOCmljcBZiSma67L5sQJE0TkFd//WlRBXng6C1E+Bw6Y6laAKVMqoKJA+kAIIUSZ5ebC4HvTOH4cfBpvYG3bnhxPPU67eu2YeedMJvSa4O4iXhZvbMJUVKOIRrx141scf+44/Zv3B8ys4HUe+RMq9AwrVsALhZhMPwAAH5JJREFUE9PdXEpRmbz4dBaibF54AbKzTQe97t0rYAdSAyGEEGX24ouwLjYMQk5hGXIvN7e5gR8e+oHfx/zOiC4jCPKr3NmbXc0bmzAVJzww3DafRlZuFmf8tqHvfhBULpP/FUK/Z+bwx+k/3FxKURlcNgqTEJ5s5Ur49lsIC4N//rOCdiIBhBBClCojJ4Mvfv+CqyKvYsvS7kyeDH7+Fm77+2e8OeJH2tVr5+4iulTdO+sS0jqEhMgEdxfFpQL9Atn4+EZWDVjF034fsmvWU8T+5wHaJd/KDQNgyq1TqtxnKQpIACGqvJwceNY6Qt3f/mZG96gQ0oRJCCGKtefMHj6O+5jPt39OcmYy3VLeYPO/TXXw9I99eOwx726qVJxGTzcCICG2agUQAEop+jfvz85P+zMi5AyzptWFBd8QW6MvNe+qaUuXZ8nz6JGyRNlJACGqvLffNuNVt2xZEEhUCKmBEEIIO9l52Sz6YxEfbf6I2MOxtvVtkp9i+4cT0Bpee82FE3oKt5kxpS4XkuDrr2tQ878buPh0MISDRVvo9FEnukZ35fGuj9OnSR+U3GjzehJAiCpt504zXCvAxx9DYGAF7kyGcRVCCDvvrX+PP//0ZwBC/UMZetVQOl18kRcfa0NONowbVzDrdFWV9nsalnQLZLi7JBXLxwdmz4akJFizJpi+feGnn+Bijc3sPr2bXad3MWfHHNrUacOIziNond3a3UUWl0F+6YgqKzfX3NXKyYEnnoAbbqjgHUoNhBCiGkvOSOajuI+YvX22bd2wjsPoGt2VD277gIQXErghfTrPP9qGzExzXZ48ueq3+owfGc+W67bAIXeXpOIFB8Py5dCvHyQmQkwMBJ69hv1P7+cvvf9Cg/AG7D27l4k/TeTe9fcyaP4gTl+USSS8kfzSEVXWu+9CXBw0bmyaMVU46QMhhKhmcvJyWBq/lHv/ey/1363PmO/H8Novr6G1GXmoYURDNo/ezJhrxjBjWgQPPGBGw3vqKfjgg+pRYRvaPpTwa8IhxN0lqRyhofDdd3DTTWaCuX794MTOFvzjhn9w5NkjLH1wKYOvGIxSii0nt1A7uLYt7+7Tu23njvBs0oRJVEm7dxfMaDpjBkREVMJOpQZCCFFN7Dq1i2mbpvH17q85nW7uICsUN7a4kUc6PYJFW2xp8/Jg/Hh4/33z+u23zdCt1eVeyxWzrgAgNjbWvQWpRCEhZuTDe+81wcQNN8DUqfDEE34MbDOQgW0GsnDFQiKviLR1rj518RQdP+xIo4hGPNDhAR7s8CAdozpKfwkPJQGEqHIyMuChh8xdrpEjzV2QSiEBhBCiisrJyyE5M5nI0EgAjqYc5cO4DwG4su6VPNrpUR7q+BCNIhrZ5UtIMLMUr1plJh777DMYOrSySy/cISgIFi2CCRNM8Pjkk7Btm2m2FhAAtQNq07tJb1v6/ef2Uz+sPkdSjvDWurd4a91bXFn3Su5tdy+DrhhEl/pdJJjwIPJLR1Q5zzxjLlKtWhXMPF0ppAmTEKIKSc1KZeEfCxnx7Qjqv1ufJ7970vbeDS1u4O99/86W0VvYNXYXL/V+6ZLgYePG2nTubIKHqChYsaJ6Bg+WHIuZTK4atszx84P33jOBY2AgfPQR9O4NfziYa65n454cfe4oq4ev5smrn6ROcB3+OPMHr/7yKtd+ci3Jmcm2tNLMyf2kBkJUKbNnwyefmAvVf/8LNWpU4s6lBkII4eUOJh9k4R8LWbZvGWuOriHXkmt772jKUSzago/yIcA3gP/r938Ot5GaaubcmTq1IwADBsDcuSaIqI7iOsaRvicdPnN3Sdzn0UfhyithyBDYtAm6dIERIxrRpw/4Fpoewkf50LdpX/o27cuUW6fw06GfWLxnMSlZKba+EhZtof0H7elcvzO3tLyFG1veSIPwBm76y6ovCSBElbFzp6kiBZg2DTp3ruQCyDCuQggvk5SWhEZTP6w+AEvilzB+5XjA/Jjr3aQ3t7a6lcFXDC51VmGtzY2bZ5+FkyfBx0fz2muKiROr92VR51nvllfzyulrr4Xff4fnnoNZs+DDD1uxfTtMmQJXX31pen9ff25pdQu3tLrFbv32xO3sObOHPWf2MH/nfADa1WvHjS1u5MYWN9KveT9C/KtJj3U3kgBCVAnJyebORkaGudMxYoQbCpFfAyFNmIQQHiopLYnVR1YTeziW2MOx/HHmDyb2msibA94EYGCbgWxN3MptrW7jppY3USu4llPbjYuDv/wFVq40r6+7DkaO3Mzjj3erqD/Fa9gCCJmImRo14NNP4e67YfjwLH79NZBu3WDYMPjHP8yoiaXpEt2F/U/tZ+nepaw8uJLVh1ez+/Rudp/ezeTfJhM3Ko6rG5iIZN/ZfUSGRlIjqDKbI1QPEkAIr5eZCYMGQXw8XHWVGRrQLb/h8/tAVOdbbUIIj/TKqlf4atdXxJ+Nt1sf4h9CRm7BDGetarfi88GfO73dbdvglVdgyRLzulYt+Oc/4fHH4Zdf0lxSdq+XPyCV3FuyGTgQPvtsE6tX92bKFJgzx9RePfmkqaFo0qTk/C1rt+TZ657l2eueJTsvmw3HN7DywEp+O/EbnesXND8YsWQE646u46qoq+jduDe9mvSiZ+OeNK3RVDpkXyYJIIRXs1jMnYs1a6BhQ/j+ezN8nNsKg/SBEEJUvlxLLvFn4tmetJ24hDh+O/EbX979JU1rNgUg4UIC8WfjCfEPoVfjXsQ0iyGmWQzdGnQjwDegTPvKy4Nly+A//zEdo8Fcd8eNM8O11q3r6r/Ou9lqIOSrwU5YWC7vvANjxsDEiSaA+Pe/zXn14IPwwgvQqVPp2wnwDbD1myhMa42/jz9+Pn7sSNrBjqQdfBD3AQB1guswKWYS464dB5j/H1/lK0FFGUgAIbyW1vD88/D112aeh+XLnav+rDASQAghKtGZ9DNMWDmB7Unb2XVqF1l5WXbvbzi+wRZAPNfjOUZfPZpO9TuVOWDId/AgzJ9vBqo4fNisCw42d41feqn6dpIujTRhKlmLFrBgAWzdCu+8A199ZWok5syBbt1Mk+QHH4SaNcu2XaUUPz/6Mxk5GcQlxLHu2DrWHl3LhuMbOJtxllD/UFvaeb/P49kfn6Vz/c60r9feLJHm0dlmfNWNBBDCK2kNr75qxpP294fFi03zJbeSYVyFEC6SkZPBvnP7iD8Tz96ze4k/G0/82XiiQqNY8qBpLxQWEMbs7bPJ0+ba07xmczrV70TnqM50b9SdHo162LZXWgfo4hw4AEuXmsDht98K1rdoAWPHwmOPQe3axecXSBMmJ3XpAl9+afpCvP++GVUxLs4szz1n5nS66y64446y1XIF+wfTp2kf+jTtA5iaiWOpx4gILJhhdvfp3ZzLOMfPh37m50M/2+VvVbsVe8fttdVOxB6OpX5YfZrXbE6gX+Dl/+FeSgII4XW0hj//Gd56y3Q3mD0b+vVzd6mQGgghhNMyczM5mnKUw+cPc+T8EW5udTNNapiG36+seoXXfnkN7WDigKjQgtv8QX5BfD74c5rWbErHqI52P4jK6+xZ+PVX+N//TK3uvn0F74WEmP5mDz8MN99sP/ymKJ40YSqb5s3NyExvvWVuDn76Kfz0kwlkly413/vdu5vZrfv3hx49zKR1zlJK2f7X8r1xwxuMvWYsO5J2sOv0LrOc2sXu07upEVjDFjxYtIVbv7iVzNxMFGY7LWu3pFWtVrSs3ZKBbQaWO1j3NhJACK9isZghAqdONRPUzJ0L99/v7lJZyTCuQlR7WblZJF1M4uSFkwB0b9QdgAtZF3jwmwdJTEvkeOpxki4m2eX7ashXth81NYNq4uvjS4taLWhbp61Z6prHNnXa2OV7qOND5S5rairs2AHbt5vO0L/+Crt326epUcPc+b37bnPnNzTU8bZEEW+/DddcA/36oS3FNGFatcpMijBhQqUXzxsEB5umSw8+aIYFXrLEzGz988+wfr1ZXn/dzGrdsaM53N26maVdO/MbwVlKKRrXaEzjGo25vc3ttvUWbeFcxjnb67TsNPo06cP+c/s5knLEtuTXWkSHRdsCiA82fcBrv7xG44jGNIpoROOIxjQIb0BkaCT1w+pza+tbXXOg3EQCCOE1srNNW9tZs8wFY8ECczfMY1ibMMkwrkJ4N6016TnppGSlkJqVSmpWKuczz3M2/SxnM85yX/v7iAyNBOCdde/w1a6vOJtxllMXTpG+Ot22nWsbXstvj5t2PyH+ISzfvxyLNjca/Hz8aBzRmKY1m9K0RlO7ibCe6PYE464dh7+v/2X/Lbm5cPSoaYqUv+zfbwKHgwcvTR8YaO7u9ukDt9xihmMtyw8xYXXNNXDffeaLKs/6ORb+ali1quB9UaroaHjiCbOkpsIvv5hA4uefTQCc39QpX1AQtG4NbdpA27ZmadMGmjY1fXWcvc/no3yoG1LQXioiMIIVw8zIAdl52Rw5f4QDyQfYf24/B84doEt0F1vaoylHSUxLJDEtkU0Jm+y2GxUaReKLibbXHT7oQPKFZJoeaEpkaCT1QupRK7gWtYJq0b95f9uNiJTMFJIuJlErqBY1g2q65BpRXnJZEF4hKQnuuQfWrTMXhsWLTRW6R5EaCCEcUkrVBmYCNwFngD9rrb90kE4B/wQet66aCbyktb60LU8hOZYcDiYfJD0n3W7JyMlg0BWD8PMxX3Wfb/ucvWf3kp6TzsWci7bgIDUrlf7N+/Nqv1cB2HlqJx0/6ljs/jrX72wLIBIuJLD55Gbbe77Kl6iwKKLDoulQr0PBeh9flj64lDrBdWgY0ZDosGh8fRy3ASppEqy8PPMD6vz5giU52VwjT560XxITzfr87llFBQRA+/Zm0s1OncxEX127miBCXKZ+/UxwcN996OwFgKLxgnngk2a+xB580LzvEe1vvUtEhBkGduBA8zolxXTAzg8i4uJMoPz772Ypys/PjNrYuDE0amQCinr1TL+KunXtn0dEmP8HR/cFA3wDaF2nNa3rtHZYztf7v86frvkTx1OPcyz1GMdSjpGYlsip9FOE+BX8j2ut2X9uP1l5WSQcT7hkO28NeMsWQPx44Efu/7qg2UVYQBjhAeGEBYQRFhDG6uGrCQ8MB+DdX9/laMpRwgML3g8LCCPYL5iWtVvSrYGZoyUrN4tD5w8R7BdMsH+wU58BSAAhvMCWLTB4MBw7Zv7pFy0yN3c8jvSBEKI404BsIAroDHyvlNqutd5VJN1oYDDQCdDASuAg8FFJG9+RuIOW77cBFGhlffQBrTj+fAI1AmtiscCnG77hl8O/2L2fn6dGbltOXmH6WKWfr0VgWivCAyIICwg3j/4R1AysRc3AWlw4WZ/4DJP2pvDn6XrdSMJ8a7J/1xGu7dyTvDxFTo65+794sXk0r28jKRe2W9/LzDSTX6anm8f8pejrtLSCYCE1tewHv2FDaNnSfunQwdyV9XffDcyqr18/mDcPfWMmEEzTL+bAN7MLxsGV4MElatSAmBiz5EtNhb17zRIfb5Z9+8zviNOn4cgRszjDzw/Cwx0voaEmwMhfgoIKv/YjMLAxQUGNCQzsQbNAaO0PfgGm/9D//mcefXxgcc/TbNi2lkZtojmfdZaU7HOk5aZwIfs8jXOv59AhkzbldBhNfa4jJfs8KVnnSUvLI03lAOdAneViaiC5fibgmb95OXEJG0FpQNs9Dr1qKJ8OnolSsPv0Prp+3Nm8ryylHI1Cx8X5j0iIymWxwIcfwosvmi/aHj3gm29MVaZHkgBCiEsopUKBe4AOWus0YK1SagkwDJhYJPmjwLta6+PWvO8CoyglgGh9sg3/eW26w/d2vbHd9vyvvMBfeYFZNGc+pr9Bb07zMrtZR10aPGHSRVGHJcwoYY8nOMIJwDRrjwLWoXiVXtb8GcxmI+cI4hG623J9za+Ek1PSn2InqZj8I+iAL2nU5Dx/IpkWaBTa2kLm0uecwCy/mO305F78MZHIDt4kma5cxV+ojalJOcRjHOUBp8sJOMzfglmA+ZF8mt7s5uUybbM5s2jCfLv8dVlHe0xNUQZRbGR2mbbpKH8QSXTnEVuaX/maHMKd3mZx+fOPczjvYyEQRS5kWj//G28sU7mropgK3HYE0M26FJVJIMdpxDEac5xGnKYep6nHGeraHvOXVCLIyQ0gOdnU9FUMBYQDjvtEzLR7dZt1cSz67cKv/ldsui+ti9EByC1SntJJACE80rFjZuzn/1nP/5EjYdo0D69al2FchXCkDZCntd5baN124HoHadtb3yucrr2jjSqlRmNqLGhDGwIcjFhUnFDSqcU5FJoIUglAE0ImUSSi0ESSVabtAdTlHNexHn9yqEMuAfgQwUUGsRg/cvEnh1AiCSjDZACNOMkaehPKRWqRzDGmkEcEh2hhCwC28zbJXENxX/ql/RUaXzQBdvkL1pXFpfm13d/qU+ZtOsqvixy/y9umyW/BvhrGgl+Ztlta/i48V6YyiooVRBatOEArDjiVPosALhDucEknhCwCySSILAIvWYquzyaAPHwdLrn4OfWeBR/bbQLXL87fAJUAQniUvDzTSfrFF027xjp14KOPYMgQd5fMCVIDIYQjYUBKkXUp4PAWb9G0KUCYUkoV7QehtZ4OTAfodnU33fdX+1loS9LXF2b4mf9TnVcbnducvj7wkr9P/rbR2U1K2sSl2/SBq9b9QkxMjDW/Ke6gwILrgSXL+eYB+XwCB9ueN7HmVwHnbTcqrsq2lB4lFOEov/LfCD5mXbNcC82K6TdR7DYd5P9lLTS9YS4AdfM0fXPLWFDfvuBXJL9PX/D/OwBBWtM3u4zbLC5/YEGNS89yfE6O8quA8/D993DvvaYaPV9QkJl2Ob8RfzUVGxtLTOF2Rx4q0LpU9ATrnnI8nL0HKgGE8Bhr1sAzz5jOUGCGDJw+HerXd2+5nCadqIVwJA3ToqCwCOCCE2kjgLTSOlGjwCewfP93ylehfO2/MZVSqMDy1yQWl7+8ZSwpv0/AZW7TQX4fP5/L+nVgy1/oZr+j41wWXvs5BQWZO2NBQeisLFRgoO21EN5MfukIt/v9d3ODpm9fEzw0amRmo/z2Wy8KHkCGcRXCsb2An1Kq8FAlnYCiHaixruvkRDohPN+qVWa0pWXL4L//5fBjj5mah2XLzPpVq9xdQiHKTWoghNv89hu88YaZHAbMpDETJpglpPhRDD2X1EAIcQmt9UWl1ELgVaXU45hRmAYBPR0knw08r5RahmmY8wIwtdIKK4SrFJ7nwTra0pGwMJrnN1GxDvEqQ7kKbyUBhKhU6enmejl9uplFEkxN7qhRMH68GZfZa0kfCCGKMxb4FDgFnAXGaK13KaX6AMu11mHWdB8DLYD80dtnWNcJ4V02bSo5OMifJ2LTJgkghFeSAEJUuNxcWL3aXCvnzy8YxzwiAsaOhWefNRO5eD0JIIRwSGt9DjO/Q9H1azAdp/Nfa2CCdRHCe01w4hTu10+CB+G1JIAQFSIlxdTgLltmJlI6fbrgveuug9GjTe1taKj7yuhyMoyrEEIIIaoBCSCES5w7Bxs2wK+/msDht98Kfk8DtG5tOko/8ABcdZX7ylmhpAZCCCGEENWABBCiTCwWM/37jh1m9KQdO2DDhms5dsw+nZ8f9O5tJtscPNgEDVX+xrx0ohZCCCFENeCSAEIpNQ4YDlwFzNNaDy8l/XPAS0Aw8A2mQ12WK8oiLk9enmlulJgIJ07AoUNw8GDB44EDcPFi0VwhBAbCNddAz57QqxfExJg+DtWKDOMqhBBCiGrAVTUQCcDrwM2YoKBYSqmbgYlAf2u+RcD/WdeJy6Q1ZGVBRoaZ+DItDc6fN30SUlIKnhd+TEoyAUNiogkeLKVMwlm/vqlR6NjRPGZlxTF8eDcCAirnb/RYUgMhhBBCiGrAJQGE1nohgFKqG9ColOSPAjO11ruseV4DvsCJACL5+EX+O/43tFbW/ZqBwu1ea9DYvwazzpnXxW630OsTCZnsmLvd+p6yvW/32sF+LFqRZ1Hk5ilyLYrcPJ+C19Z1edb1tteF0ubmKbJzfcjM8SEj25fMbF8ysq3Pc3xt6y5XvYgs6tfKpH7NTJpHptMi6iIt6l+kRVQ6zSMvUjs8xy79zp07Cfj+WDFbq0b27gWkD4QQQgghqjZ39IFoD3xb6PV2IEopVUdrfbakjAeTQrnvX90rtHDO6VR6EjcKIIsgMgkmg1AuUpPz1CCl2McokqhPIvVJpB6n8U/NhVTgiHP761Chf433sfhJ1yIhhBBCVF3u+KUTBqQUep3/PBwzwZAdpdRoYDRAuE9rrq8Vi0KjlLnFr6z1DcpaB2DW59dB5L9f8J5dWut2HKVFadu27dNqtEXj46OK3Y613qFQGU0aH2XBT+XhS555VBZ8Vf7zPNtzP/JsaQvey3+dS7BPFkE+WQT6ZBPkk0VwoeeBPjn4qlLaIDkUCrTkPC3LnDMvNxdf+dEMQG5YGEe6dSMhNtbdRfEYaWlpxMrxsJHjIYQQwtuV+qtPKRULXF/M2+u01r3LuM80oHD32vznFxwl1lpPB6YDdOvWTS+Niynj7lwvNjaWmPzp6IUcjyLi5XjYkfPDnhwPIYQQ3q7UAEJrHePife7CtAFaYH3dCUgqrfmSEEIIIYQQwv1c0ttTKeWnlAoCfAFfpVSQUqq44GQ2MFIp1U4pVQv4G/CZK8ohhBBCCCGEqFiuGi7mb0AGZiSlh63P/waglGqilEpTSjUB0Fr/ALwNrMJ00z0CvOKicgghhBBCCCEqkKuGcZ0ETCrmvaOYjtOF170HvOeKfQshhBBCCCEqjwxYL4QQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQokIopWorpRYppS4qpY4opYaWkHa8UmqnUuqCUuqQUmp8ZZZVCCGE8/zcXQAhhBBV1jQgG4gCOgPfK6W2a613OUirgEeAHUBLYIVS6pjWen6llVYIIYRTpAZCCCGEyymlQoF7gJe11mla67XAEmCYo/Ra67e11lu01rla63jgW6BX5ZVYCCGEs7yqBmLz5s1nlFJH3F0OoC5wxt2F8CByPOzJ8bAnx8OepxyPphW8/TZAntZ6b6F124HrS8uolFJAH+DjEtKMBkZbX6YppeIvo6yu4imfraeQ42FPjoc9OR72POV4OPXd4FUBhNa6nrvLAKCUitNad3N3OTyFHA97cjzsyfGwV42ORxiQUmRdChDuRN5JmBryWcUl0FpPB6aXt3AVoRp9tk6R42FPjoc9OR72vO14SBMmIYQQZaaUilVK6WKWtUAaEFEkWwRwoZTtjsP0hbhda51VMaUXQghxObyqBkIIIYRn0FrHlPS+tQ+En1KqtdZ6n3V1J8BRB+r8PCOAiUBfrfVxV5VVCCGEa0kNRPl4VLW5B5DjYU+Ohz05HvaqxfHQWl8EFgKvKqVClVK9gEHAHEfplVIPAW8AN2qtD1ZeSV2qWny2ZSDHw54cD3tyPOx51fFQWmt3l0EIIUQVpJSqDXwK3AicBSZqrb+0vtcHWK61DrO+PgQ0Ago3W5qrtX6ycksthBCiNBJACCGEEEIIIZwmTZiEEEIIIYQQTpMAQgghhBBCCOE0CSBcQCnVWimVqZSa6+6yuItSKlApNVMpdUQpdUEptVUpdau7y1WZlFK1lVKLlFIXrcdhqLvL5C5yPhRPrhfVg3zOch3IJ98NBeScKJ63XTMkgHCNacAmdxfCzfyAY5hZZmsALwMLlFLN3FimyjYNyAaigIeAD5VS7d1bJLeR86F4cr2oHuRzlutAPvluKCDnRPG86pohAcRlUko9AJwHfnJ3WdxJa31Raz1Ja31Ya23RWn8HHAKudnfZKoN1zPt7gJe11mla67XAEmCYe0vmHtX9fCiOXC+qB/mcDbkOyHdDUXJOOOaN1wwJIC6DUioCeBV4wd1l8TRKqSigDSVMGlXFtAHytNZ7C63bDlTXu0x2quH5cAm5XlQP8jkXr5peB+S7oQTV9Jyw463XDAkgLs9rwEyt9TF3F8STKKX8gS+Az7XWe9xdnkoSBqQUWZcChLuhLB6lmp4Pjsj1onqQz9mBanwdkO+GYlTjc6Ior7xmSABRDKVUrFJKF7OsVUp1BgYA77u7rJWhtONRKJ0PZqbZbGCc2wpc+dKAiCLrIoALbiiLx6jG54Od6na9qKrke8GefC84Rb4bHKjm54SNN18z/NxdAE+ltY4p6X2l1LNAM+CoUgrMXQZfpVQ7rXXXCi9gJSvteAAocyBmYjqK3aa1zqnocnmQvYCfUqq11nqfdV0nqne1bHU+H4qKoRpdL6oq+V6wJ98LTpHvhiLknLATg5deM2Qm6nJSSoVgf1fhRcxJMEZrfdothXIzpdRHQGdggNY6zd3lqWxKqfmABh7HHIdlQE+tdbX8oqju50Nhcr2oHuRzvpRcB+S7oSg5Jwp48zVDaiDKSWudDqTnv1ZKpQGZnv6BVxSlVFPgCSALSLRG0gBPaK2/cFvBKtdY4FPgFHAWcwGorl8Qcj4UIteL6kE+Z3tyHbCR7wYrOSfsefM1Q2oghBBCCCGEEE6TTtRCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHCaBBBCCCGEEEIIp0kAIYQQQgghhHDa/wO/jsL3o9WIqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff94bf575c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200) # -5到5之间均匀分布的200个数\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\") # 阶跃函数\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\") # 逻辑函数\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\") # t双曲正切（tanh）函数\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\") # relu函数 m:megenta洋红 \n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Activation functions\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "#plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Derivatives\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAEMCAYAAACfoCGmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xu8XHV97//XJ9nZOzcCgcSkAZJguATxZ1DAG6KopaLWGi7nVwXBCxov5ecV8+N4RCm1ijnFWhBRKpeKQvtTLpVatbTqKZRjRTheoA0okHBrApJNkp3L3tk7n98faw1OJrP3nlnzXff38/GYR7Jn1qz1mdmzv/Oe76zPWubuiIiIiEixTMm7ABERERHZm0KaiIiISAEppImIiIgUkEKaiIiISAEppImIiIgUkEKaiIiISAEppGXAzNaZ2XkZbOdCM7s3g+1MMbOvmtnTZuZmdmLa25yknmvN7B9y2O7S+PEfm/W229TS8XNgZifGdc+bYJnTzUzH55EJaWxLvZ5cxrbJFKmuTmoxs38ws2szKiko03HS9mRmLwR+BvzE3Y/v8r4XAqe7+/Nbrp8PbHP37YFqXAo8DBzn7j9run42MODuT4fYzgTb/0PgJuBE4CFgk7uPpLnNeLsnAj8C5rv7b5uu35fotfxM2jW01LOUNr+HPHTzHJhZP7A/sNHHGQDM7HTgW+5uYSuVvGhs62j7Gts6UKS6OqklDnG/dfd3ZFZYIH15F1BA7wG+DJxtZke6+3/2ukJ3f6r3sjrazhAwlMGmDgX+y93vzGBbk3L3zXnXkLdunoP4TWdDiuVIMWlsm5zGtg4Uqa4i1ZIKd9clvgAzgGeAFwBXAX/RZplFwDeBp4HtwM+BVwPvALzl8o74PuuA8+L/3wDc2LLOKcCjwEfin08GbgcGgU3AD4Ajm5Zv3c6P4+svBO5tWe8F8bqHgV8Bb266fWl8/9OA2+LH8x/ASRM8R9e2bHtdfP2PgS+1WfYfmn7+MdGbxGeB3wJPAn8BTGlapj++fX1c80PAB5tqbb5cO852BoAvAhuBncBPgFc03X5ifP/XAv8eP+6fAS/q8vXS0fMHPA/4LrA1fsw3AAubbj8O+Kf4OdkC3AG8rOn2Tl4zrc/BK+PHPQRsjh/n81se/7ym5c+On/PtwD8AfwJ4yzbfBNwdP6cPA38O9Of9d6tLR69VjW0a27p9zXyqqdYNwNcnePyzgK8TjTcbgf9ONI5c27TMunid1xKNhY8CfwzsB/xtfN9fA3/QUscr48eyM173X9I07rSpZWZ8XaOWT7TWUqZL7gUU6QKcBfwi/v+J8R/atJYX4q+Bf4tfOMuAU4kGshnxH+VaYGF8mdH04mwMZG+MX2z7Na331cAo8Rs30cByGnAY0aD6/wG/abwwid7UHXhdvJ394+svZM+B7CNEb/pnAIcDFwFjwNHx7Uvj9awlegM+DPgbokF69jjP0b7An8Z/YAuJpueh84Fsc1zH4cD/HT/utzYtcwPwWPz4nxs/N2cDU+Pn2olCz0Jg33G281fAf8XP9ZHAX8d/sL/X9Lt14Kfx+pcTvVn8J/EuAPFyDlw4wetl0ucP+D2iQfvzcS0vAG6Ntz0lXuY1RK+9I+NavkT0Jjavi9fMs88B0Qz5INHrcVm8zjOI3wxpCWnAS4DdwP+Ify/vjR+DN23vdUSvpXfG63w1cD9t3ux1Kd4FjW0a27ob206Ln983AouBY4FzJ3j8XyEKdCcBRxGFrs3sHdI2AR+Ifx+XEL1e/jF+Hg4l+gDxJDA9vs+BwLZ4/UcCf0gUGC+ZoJYvA48TvYaeD3wrfizXjvd4i3zJvYAiXYD/xe8GHItfVKc13f4eok8A88a5/4U0DSRN169rWm9f/CI8p+n2rwE/mKCuWUQD0Cvin5fGf2THTrT9+IX6qZZlfgx8o2U97226/cD4uldMUM95xJ8yW9bbyUD2v1uWuQ34Wvz/w+JtnzzOdk+kZQaodTvxczUCnN10+1TgQeAzLet5XdMyx8fXHdR03VqaBqY29Uz6/BEN2v/Scr+58TIvHme9RjQQv63T10zLc7B/vP5XdfI8AtcDt7Us8zX2DGn/ClzQssxKojcIa7cdXYpzQWPbXn+b49SjsS26/aNEH8KmjXN7c12z47re0vJ7HWTvkHZD08+z47oubbpuj98/0Wz9b9hzRvIdRLN7M8epZRg4s2U7z1DSkKbuzpiZHUr0Yr4e4nenaOr/3U2LvRD4pTft2Nktdx8F/g44M97uANGnlm801bLMzK43swfNbAvRlO0Uok80nT6eOURfX/xby013EH1aa/bLpv8/Ef/7nE631aVftvz8RNO2Xkg0o/OjHta/DJhG0+N29zHgf9Pl43b35e7+pQ62OdF6jgFeaWZDjQvRJ/VGrZjZc+KOsgfMbDPRm+VziH/fnbxmmrn7JqKB6wdm9l0z+6iZHTxB/UcSPT/NWn8+BvgfLY/jeqLBeOEE65acaWx7lsa2391vsrHtW8B04GEzu8rM/lv8+5yorp82rX8b0K4b95dNywwRfR37q6bbN7bUeiRR+N3dtMwdRF8dHzpOLf00jV/xdn7VZtlSUOPA77yb6FPJI2bPNrQZgJkd7O6PNn4O4BvAnWZ2INFXTf3AzU2330r0SfG98b+jRPtT9CfYlndw3a5nb3D3+PF3G+B3s/fzM63NcrtafvambYV4fhvr6OpxN92W5IPLRM/fFKL90dodpqAxIP0NsIDoK5x1RJ8E/4U9f9+TvWb24O7vNLMvEu0D9EfAn5vZSnf/QZvFO3nepxB9FfStNrdlsvO4JKaxDY1tdPG43f1RMzuCaN+23yf6avLTZvaSOIB1Wlerds/RRLXaBOttd33lutE1kwaYWR/wdqKdHY9uuqwgSv7vjBe9B3jBBMeXGiEaDCfk7v9ONEX9VqJPnbfEaR8zO4Do08Nn3f2fPerA2oc9A3WjJXzcbbn7FqJPUK9ouekVRINiaE8R7X/VbEWX67iH6DX56nFun/RxE02Nj9D0uM1sKvAy0nnck7mHaB+N9e7+m5bL1niZVwCXuft33f0+opm0PZ7LiV4z43H3X7j75939RKKvY94+zqL/Aby05brWn+8Blrd5DL+JZ1CkgDS2BVHLsc3dd8Zj0keI9hU8imhGtl1du4AXN9U1k2h/sF79B/AyM2vOKq8geh4enKCWZ8cvM5sVqJZcKKRF3gjMA/7a3e9tvhDtAPmu+EVyPdE+F7eY2QlmdoiZ/ZGZNf7w1gFLzOxFZjZvgulh+N3XDW9kz6+tBol2NH+PmR1qZq8i2mmy+Y3wSWAH8DozWxAfJ6ad/wmcZ2ZvNbPDzewi4ASiT0Wh/RB4ffx8HGFmXwAm+optL+7+a6Idib9mZqfFz+8JZnZWvMh6ok9PbzSz+fGxk1rXsQ24ArjYzN5gZkfGPy8g2qG0Y2a21szO7eY+bVxOtEPy35nZS8zsuWb2+2Z2pZntEy/zAPA2M3uemR1H9Jprd2ym8V4zrXUfYmYXm9nLzWxJ/Pp8AeMP5JcCv29m/93MDjOz9wCntCxzEXCGmV1kZs83s+UWHfB2TYfPg+RDY1vvaje2mdk7zOzdZvZ/mdkhRGF+F1FzSWtdQ8DVwOfN7LVm9jyifRGn0Nns2kS+TPTV9pfN7EgzeyNwMdE+gnsdmy+u5aq4lpPM7Ki4tkk/YBSVQlrkHOBH3v5Aid8ClgC/H/+RvIpomv5W4D6ir4AaL8QbiTpV/oXo09dbJ9jmN4AjiDpgbmtcGX/3/sdEb6r3Er3JX0D0FVhjmVGi1u13E32i/PtxtnEp0WC2Jl7XKUQ7C/98grqSurrp8m9EO5SP+3XcBM4mesO4lGjn1muJQg7u/jjwaaKdSTcSdUG28/8SDYjXEB1G4AVEO+z+V5e1HEH0BpeYuz9B9OlzN/B9otfM5US/z8bv9F1EO7feTfTGeTXRm2Krtq+ZNrYTdZh9iygA/g3RG+fnx6nxJ0R/A+8nml05lWhH7eZlfkD0pvtqon1PfgqcDzwyQR2SP41tvavj2PYM0WvndqLn9zTgVHd/eJzlz4uX/Q7Rfne/JDr0x84u69pD/Ly8nmifvp8T/Q5uIDqsxnjOi2u4Of73XqLGp1LSGQdEREQkmHimdT3wP909jdnN2lDjgIiIiCRm0SnHjiSaYd+HaMZvH6JuX+lBsK87zexcM/uZmQ3bBCcyNbO3m9ndZrbFzB4zszXxzq0iIrnRGCbSk48C/4doH74FwCvd/bF8Syq/kPukPQF8hug744nMBD5M9H34S4hafNsdnkBEJEsaw0QScPf/4+7Huvs+7j7X3V/t7nfnXVcVBPv05+43AZjZscBBEyx3RdOPj5vZNxm/LVlEJBMaw0SkaIowRf9Kok6itsxsFbAKYPr0gWMOOjitg0V3z70Ps2IdIipUTWM+hbHRqdjY7skXnsDUvimMjfa2jpBUz8SKVg/A+kfX/dbd5+ddxwTGHcNax6+DCzR+QXZj2GiHX9pM8anstrGUq5ncmEf19rkxavk013mbP8M+m8JouxuCbDDZcWCnTTF27e78OUr76eybYoxmMIZZh02XvY5fuYY0M3sn0Ylb3z3eMu5+JXAlwGGHL/Zv/3NxulGfWPsRFi0v1mGiwtU0xnWDL+G2q1/KohvH67qe3KkfPo6b1twVoJ4wVM/EilYPwHouW593DeOZbAxrHr8OP3yx3/ovxfpQt27teSxdfnEm2/r2lhdNusyyR1fy4MG3ZFDN5G7bsJwzth/N9TPTOKpHZ9Y9tud7+0dmL+GSoXT/HAYe6e7kDx889EAu/c3jXW9nn/XpvJefc8KBXHX74+z74PDkC/eof+3ku9yt58s9/cJyO06ama0kOijd63s5X5yk56y5d3LSu37C/auX5F2KSOFoDOvO6XPuybuErpy0cC1zpvV0mK+eLT0o+zOuDS9udxzt8LYuSfcMTpuXDbB52UTHXO7dyPJx94oIJpeQZmYnA38NvMndS3vi0zo4a+6d3Lzyi9y/egk7nn9g3uWIFILGsGTKFtQgCmt5WnrQU5mHtaoENaD0QS3kITj6zGw60ekXpprZ9HZt6Wb2GqKjn5/m7j8NtX1J180rv8imP9mmoCaVpTEsGwpqySioJVfmoBZyJu2TROdcOx94W/z/T5rZYjMbMrPF8XIXEJ0K4x/j64fM7HsB65CUXLfiGjb9yTaeOO2QvEsRSYPGsIwoqCUz0J/tPo0Kap1LK6gFC2nufqG7W8vlQnd/xN1nu/sj8XKvdve++LrG5fWh6pB0XbfiGk56108U1KRyNIZl6/Q595QurBUhqGlGLbksglrosKYTrEvXGg0FCmoi0isFte7lEdSyCGtbl1gmDQVpCxnUFNIkEXV+ikgoZQxqeYc1dX4mV6bOT4U0SUydnyISStmCGuQ/q6bOz96UIagppEnP1PkpIiHMnbo97xK6lndQA+2n1ossvv7shUKaBKHOTxEJQTNqySioJVfkoKaQJsGo81NEQlBQS0ZBLbmiBjWFNAlKnZ8iEoIO0ZFM1kFtd/9udX6mSCFNglPnp4iEUsaglndYU+dncll0fnZDIU1S0ej8HF7Yr4YCEelJ2YIa5D+rps7P3hQlqCmkSaqW7fekOj9FpGcKaskoqCVXhKCmkCapU+eniISgoJaMglpyeQc1hTTJhDo/RSQEBbVkFNSSyzOoKaRJZtT5KSIhqPMzGZ3zM7m8gppCmmRKnZ8iEkoZg1reYU2dn8nl0fnZl+nWKmT36Bj9fX/F7tExpvRNzbucUjlr7p2ctfJOTuHDLP7HUWbc+3jeJVXeD8+8ipGZk59y5/uDwHsmX1//9pm85pvn9F6Y5GZsdIxpUy9lbHSMqSUew06fcw/f3vKivMvoykkL13LbhuW5bb8R1NY9Nj+zbQ4vHmHgkf7E939w96cZY+uky31gHXDw5OvrG9uHFU9cmKiWzcsG2PfB4UT37ZZm0hLaMbiFKfYQOwa35F1Kaemcn9npJKDluT7J3tDgEGYPMTQ4lHcpPSvbjBrU9+vPpDoJaN0Yndrb+rKaUVNIS2D36BgjQ9swc0aGtrF7dCzvkkpLnZ8i2RsbHWP71u2YOdu3bmesAmOYgloyZQpqRZNFUFNIS2DH4Bbw+AdHs2k9UuenSLaGBof2GMOqMJsGCmpJKagll3ZQU0jrUmMWrZlm03qnzk+RbDRm0ZpVZTYN1PmZVFU7P7OQZlBTSOvSHrNoDZpNC0KdnyLp22MWraFCs2kNZQxqeYe1Knd+pi2tzk+FtC60m0Vr0GxaGI1zft6/eokaCkQCazeL1lCl2bSGsgU1yH9Wrcrn/MxC6KAWNKSZ2blm9jMzGzazaydZ9iNmtsHMNpvZ1WaW/0myJtF2Fq1Bs2lBqfNTslb18QvGmUVrqOBsGiioJaWgllzIoBZ6Ju0J4DPA1RMtZGavA84HXgssBZ4L/GngWoKaaBatQbNpYanzUzJW2fELJp5Fa6jibBooqCWloJZcqKAWNKS5+03ufgvw9CSLvh24yt3vc/dB4M+Ad4SsJbQJZ9EaNJsWnDo/JStVHr9gklm0horOpoGCWlIKasmFCGp5nXHgKODvm37+BbDAzA5w9z0GSDNbBawCmD9/Hk+s/VR2VT5rM9OnXYR1cMaJ4S0jbH76fcCc1KtqZ9fOBTyxdnUu224nRD2vBY4+eRZbjn0F0wZ7O8rz3IWzOHX1cT2tI6Ss6vn+YPh1ZvU8fv9DmWymG4nHr3VrL8iuyj1spr/vzzoaw7Zt3sXg0x8gjzFseOdC1q09P7X1Hxv/Ozg2s6PlB0b2Y9mjK1OrpxPLgC27pgOw/+6ZnLH96OyL2B+GR9rHhQVTB/jY7MDNXs+L/pky8rt5pHMfCLsJgHe+9qB4O5N9eknu7qt6u39eIW02sLnp58b/96HlU6y7XwlcCXDY4Yt90fI1mRTYbNtTg4xs3dXRsma72PeAzzFr/tyUq2rvibWryeM5Gk+oehYB1w2+nJv+14s5Ys36xOs5dfVx3LTmrp7rCSWzejo41VO3ivQ8ZizR+HX44Yt96fKLMymw1eanNrN9S+dj2NwDPse+8/dNuaq9rVt7Plk8R0uho1NJLXt0JQ8efEvq9XTsobdw/cyf57Ptme1PI/Wx2Uu4ZCj5mDyZXk4lNZlLf/O7UxLusz69oNaLvLo7h9jzY1rj/2HP+xBAJ/uitdK+aelQ56cURGnGL+hsX7RWVd03rVkZv/6cM21nrtuvcudn2idnTyqvkHYfsKLp5xXAxtavCoqgo33RWmnftFSp81NyVprxCzrcF61VhfdNa1bGoKb91NJTxKAW+hAcfWY2HZgKTDWz6WbW7ivVrwPnmNnzzGwu8Eng2pC1hJBkFq1Bs2npUuenhFa18QuSzaI11GE2DRTUksrjwLdZKFpQCz2T9klgB1F7+tvi/3/SzBab2ZCZLQZw9+8Da4AfAevjy6cD19KzRLNoDZpNS506PyWwSo1fkHAWraEms2mgoJaUglr6gjYOuPuFwIXj3Dy7ZdkvAF8Iuf2QeplFaxgZ2saMuXOY0jc1UFXS6qy5d8K74DZeyqIbH867HCmxKo1f0NssWsP2rduZPXc2U2swhjWCWicNBUVx0sK13LZhea41DPSP5rr9tDSCWt4NBTot1Dh6mkVr0GxaJnTOT5G99TSL1lCj2bSGss2q1fWcn1nJe1ZNIa2NELNoDdo3LRvq/BT5nRCzaA112TetWdmCGuT/9WcenZ9ZyTOoKaS1EWQWrUGzaZlS56dIoFm0hhrOpoGCWlIKamEppLUxNhy23Tf0+mRi6vzcW//2zo6wntf6JKyRnWHHnNDrK4u5U8PMRmapqkFtypTZky/Uhans0/V98ghqeZ1xoNDmHLSgo+WKdnR/+Z3rVlzDdYtfroaC2Gu+eU5HyxXtjAySzPyD9z4yfDtZHeG/zE6fc0+pmgmgGA0FSw96qu0ZCpI6ePEnJ11m4JF+PnjogXucSSC0rUss02YCzaRJZTUaCjSjJiK9OH3OPaX7+rOqM2oTGV48wu7+3alvZ+sSy2xWTSFNKk2dnyISShmDWt5hLY991Kp0hgKFNKm85s5PnzEt73JEpMTKFtQg/1k1nfMzOYU0qY2bV36R0efsVueniPREQS0ZBbXuKaRJrRwy42l1fopIzxTUklFQ645CmtSOzvkpIiEoqCWjoNY5hTSpJXV+ikgI6vxMJo+glkVYC935qZAmtaXOTxEJpYxBLe+wps7PySmkSa3pnJ8iEkrZghrkP6umzs+JKaSJoHN+ikgYCmrJVDWo9UohTSSmc35KmkY13NaGgloyCmp706gh0kSdn5Kmsp0DUpJTUEtGQW1PCmkiLdT5KWlSUKsPdX4mU9XOzyQU0kTaUOenpElBrV7KGNTyDmtV7vzshkKayDjU+SlpUlCrl7IFNch/Vq3KnZ+dUkgTmYQ6PyUtCmr1oqCWTJ2DWtCQZmb7m9nNZrbNzNab2RnjLDdgZl8xs41mtsnMbjUzvQNKYanzs/ryGr++veVFCms1oqCWTF2DWuiZtMuBEWABcCZwhZkd1Wa5DwEvA14ALAKeAS4LXItIUOr8rLxcxy8FtfpQUEumjkEtWEgzs1nAacAF7j7k7ncA3wHOarP4IcAP3H2ju+8E/hZoNxiKFIo6P6upKOOXglp9qPMzmbp1fpq7h1mR2QuBO919RtN15wGvcvc3tSx7LPBXwH8j+hT6NeBJd/9wm/WuAlYBzJ8/75i/+cangtQbwq6dC5g2fWPeZeyhaDVVtZ6nx2bxzNZZDGzo7Y937sJZDG7Y1nM9oRStHoBVHzr7bnc/Ns1tZDF+zZs/75jLvv65juqZO3V7wkfSneGdCxmYviGTbXWirvUMjs3saLmBkf0Y7n8m5Wo6s2XXdAD23z2TTVOyeb02Gx7pa3v9gqkDbBwbTmWbU0a6n9c694w/7mn8av8ok5kNbG65bjOwT5tlHwAeAR4HxoBfAee2W6m7XwlcCXDY4Yt90fI1oert2RNrV1OkeqB4NVW1nkXxv6fc8mEW/+MoM+59PNF6Tl19HDetuavnekIpWj0ZSn38eu7hS/3Bg2/puKAsZlnWrT2fpcsvTn07naprPUvpbBZ12aMr6eY1lLbbNiznjO1Hc/3Mn2e/8TjXrnts/h5Xf2z2Ei4ZWp/aZgce6U9t3e2E3CdtCJjTct0cYGubZa8ApgMHALOAm4DvBaxFJBPq/KyMwo1f+uqzXsr21SfU9+vPLIUMaQ8AfWZ2WNN1K4D72iy7ArjW3Te5+zDRTrcvNrN5AesRyYQ6PyuhkOOXOj/rpYxBbc60nXmXUOmgFiykufs2ok+UF5nZLDM7HngzcF2bxe8Czjazfc1sGvAB4Al3/22oekSypM7Pciv6+KWgVh9lDGqaUUtP6ENwfACYATwJ3AC8393vM7MTzGyoabnzgJ3Ar4GngDcApwSuRSRT6vwsvUKPXwpq9aHOz2Sq2PkZsnEAd98ErGxz/e1EO+Y2fn6a6DhEIpVy1tw74V1w07IXc8Sa9HZelfDKMH59e8uLSvfmLcmdPueeUoXzRlC7bcPy3GoY6B/NfJvDi0dSayjQaaFEAtM5PyVNZXrTlt6VMZTnPatWpXN+KqSJpESdn5IWBbV6UVBLpgpBTSFNJEXq/JS0qPOzXhTUkil7UFNIE0mZOj8lTQpq9ZHVmShCUlDrjUKaSAbU+SlpUlCrD3V+JlPWzk+FNJGMNILa/auX5F2KVJCCWr2UMajlHdayDmohKKSJZEidn5ImBbV6KVtQg/xn1fLo/OyFQppIDhqdnz5jWt6lSMUoqNWLgloyZQlqCmkiObluxTWMPme39lOT4NT5WS8KasmUIagppInk6JAZT6uhQFKjoFYfCmrJFD2olSqkDY/pqyGpHnV+SpoU1OpDnZ/JFDmolSqkTdm1m1Nu+TDXDb4871JEglLnp6RJQa1eyhjU8g5rRQ1qpQppAEesWc9tV79UQU0qR52fkiYFtXopW1CD/GfVitj5WbqQBrDoxoe57eqXctYv3pl3KSLB6ZyfkhYFtXpRUEumSEGtlCENoqC2/+WzFNSkknTOT0mLOj/rRUEtmaIEtdKGNIAZ9z7O/pfP4pRbPpx3KSLB6ZyfkiYFtfpQUEumCEGt1CENoqB2xJr1CmpSSer8lDQNjs3MuwTJiDo/k8k7qJU+pDU0gpoaCqRq1PkpadKMWr2UMajlHdbyDGqVCWmgzk+pLnV+SpoU1OqlbEEN8p9Vy6vzs1IhDdT5KdWmzk9Ji4JavSioJZN1UKtcSAN1fkq1qfNT0qLOz3pRUEsmy6AWNKSZ2f5mdrOZbTOz9WZ2xgTLvsjM/tXMhsxso5l9KGQt6vyUKlPnZ3hFGr/ypqBWHwpqyWQV1ELPpF0OjAALgDOBK8zsqNaFzGwe8H3gq8ABwKHAPwWuRZ2fUmnq/AyuUONX3hTU6kOdn8lkEdSChTQzmwWcBlzg7kPufgfwHeCsNot/FPiBu3/T3Yfdfau7/2eoWlqp81OqSp2fYWQxfo15+fYuUVCrlzIGtbzDWtpBLeSocTgw5u4PNF33C2CvT6LAS4FNZnanmT1pZrea2eKAtexFnZ9SVer8DCKT8eu2DcsDlJotBbV6KVtQg/xn1dLs/DR3D7MisxOAb7n7wqbr3gOc6e4ntiz7APAc4CTgV8Aa4Bh3P77NelcBqwDmzZt3zGcv+EJPde6aOwBzRjlkxtM9rQdg184FTJu+sef1hFS0mlTPxELX8/COA+h7cgq2Y1ei+89dOIvBDduC1RPCqg+dfbe7H5vmNjIZv+bPO+bPr/rLZ2+bM21n+AfSpYGR/Rjuf6ajZedO3Z5yNTC8cyED0zekvp1O1bmeTg903M1rKG1bdk1n/90z2TQl/dfqeIZH+vb4+YOnv7Wn8atv8kU6NgTMabluDrC1zbI7gJvd/S4AM/tT4Ldmtq+7b25e0N2vBK4EWLr4EL9pzV09F7rj+Qey6U+2cd2Ka3pazxNrV7No+Zqe6wmpaDWpnomFrmcRcNYv3on981wW3fhw1/c/dfVxhPgbK6HUx68lhz3Xr5/58z1WlPcMwLJHV/LgwbciqNxAAAAb9UlEQVR0dZ80Z1rWrT2fpcsvTm393apzPUvpbBY1yWsoVQ+9hda/s0zNhHWPzQ+2upBfdz4A9JnZYU3XrQDua7PsL4HmKbzG/y1gPeNS56dUmTo/E8ll/NLXn1JkZfzqswgz1CG/+gwW0tx9G3ATcJGZzTKz44E3A9e1Wfwa4BQzO9rMpgEXAHe4e2Zzpur8lCpT52d38hy/FNSkyNT5mUyooBa63egDwAzgSeAG4P3ufp+ZnWBmQ42F3P2HwCeA78bLHgqMe0yiNKnzU6pKnZ9dy238UlCToitjUMs7rIUIakFDmrtvcveV7j7L3Re7+/Xx9be7++yWZa9w9wPdfa67v8ndHw1ZSzfU+SlVpc7PzuU9fimoSdGVLahBMWbVelG+A/ekROf8lCrTOT/L4bYNy0sX1hTU6kVBLVsKaU10zk+pssY5PxXUiq+MQU1hrT4U1LKjkNZCnZ9SZdetuIZjP3+PGgpKoGxBDTSrVicKatlQSGtDnZ9SZer8LA8FNSmy0+fck8lBjkMqW1BTSJuAOj+lqtT5WR4KalJ0ZZtVK0LnZ6cU0iahzk+pKnV+loeCmhRd2YIalGNWTSGtA+r8lCpT52c5qPNTik5BLTyFtA6p81OqrNH5uWvuQN6lyCTKGNQU1upDQS0shbQuqPNTquy6FdcwZ96QGgpKoGxBDTSrVicKauEopHVJnZ9SZQdM3abOz5JQUJMi0zk/w1BIS+iINet58JnnqKFAKkedn+WhoCZFV8agVqSwppDWg4ENI+r8lEpS52d5KKhJ0ZUtqEFxZtUU0nqkzk+pMnV+loM6P6XoFNSSUUgLQJ2fUmU652d5lDGoKazVh4Ja9xTSAlHnp1SZzvlZHmULaqBZtTpRUOuOQlpA6vyUKtM5P8tDQU2KTJ2fnVNIS4HO+SlVpc7P8ihjUBscm5l3CZKhMga1rMOaQlpKdM5PqSp1fpZHGYOaZtTqpWxBDbKdVVNIS5E6P6XK1PlZDur8lKJTUBufQlrK1PkpVabOz/IoY1BTWKsPBbX2FNIyoM5PqTJ1fnbGd+ddQfmCGmhWrU4U1PYWNKSZ2f5mdrOZbTOz9WZ2xiTL95vZWjN7LGQdRaTOT6myKnR+ZjF+rXtsfu+F9khBTYpMnZ97Cj2TdjkwAiwAzgSuMLOjJlj+48CTgWsoNHV+SlVVoPMzk/FLQS0ZBbV6KWNQSyOsBQtpZjYLOA24wN2H3P0O4DvAWeMsfwjwNuBzoWooC3V+SlWVtfMz6/Fr3WPzcw9rCmpSdGULahB+Vi3kTNrhwJi7P9B03S+A8T6JXgZ8AtgRsIbSUOenVFkJOz9zGb/yDmpbdk0vXVhTUKuXugc1c/cwKzI7AfiWuy9suu49wJnufmLLsqcA73X3k83sROAb7n7QOOtdBawCmDdv3jGfveALQeoNYe7CWQxu2NbTOnzGNEafs5tDZjwdpKZdOxcwbfrGIOsKQfVMrOr1PLzjAPqenILt2JV4Has+dPbd7n5ssKLayGT8mj/vmE9dcVnb7Q/0j4Z4GF3bf/dMNk3ZDsCcaTtzqaHZwMh+DPc/0/Hyc6duT7EaGN65kIHpG1LdRjeKVg9kV1OnBzru9jWUpi27pvPePzqrp/GrL2A9Q8CcluvmAFubr4i/VlgDvKGTlbr7lcCVAEsXH+I3rbmr90oDOXX1cYSoZ8fzD+SRN/Rx88ov9ryuJ9auZtHyNT2vJxTVM7Gq17MIuG7w5dx29UtZdOPDwdabgtTHr8XLnuuXDK0fd9mlBz3VRblhnLH9aK6f+fNnf877ZNLLHl3Jgwff0tV90pxpWbf2fJYuvzi19XeraPVAdjUtpbNZ1CSvoSIL+XXnA0CfmR3WdN0K4L6W5Q4jer5vN7MNwE3A75nZBjNbGrCe0lDnp1RZSTo/cx+/8v7qE7SfmhRbGTs/exUspLn7NqIB6yIzm2VmxwNvBq5rWfRe4GDg6PjybmBj/P9HQ9VTRur8lKoqeudnUcYvBbVkFNTqpU5BLfQhOD4AzCBqS78BeL+732dmJ5jZEIC7j7r7hsYF2ATsjn8eC1xP6ajzU6qqBJ2fhRi/1PmZjIJavdQlqAUNae6+yd1Xuvssd1/s7tfH19/u7rPHuc+Px9vptq7U+SlVVtTOz6KNX0UIamULawpq9VKHoKbTQhWUzvkpVaZzfnYm76AG5ZtV0zk/66XqQU0hrcB0zk+pMp3zszMKaskoqNVHlYOaQlrBqfNTqqwknZ+5U1BLRkGtPqra+amQVhLq/JSqKnrnZ1EoqCWjoFYvaR/gOGsKaSWizk+pqhJ0fhaCOj+TUVCrlyrNqCmklYw6P6XKitr5WTRFCGplC2sKavVSlaCmkFZC6vyUKlPnZ2fyDmpQvlk1dX7WSxWCmkJaSanzU6pMnZ+dUVBLRkGtPsoe1BTSSkydn1JljYYCmZiCWjIKavVR5s5PhbQKUOenVNVZc+/Mu4RSUFBLRkGtXsoY1BTSKqLR+fn02Ky8SxGRHKjzMxkFtXopW1BTSKuQRTc+zJbfzlZDgUiNFSGolS2sKajVS5mCmkJaxUwbHFbnp0jN5R3UoHyzaur8rJeyBDWFtApS56dIAbllujkFtWQU1OqjDEFNIa2i1PkpUjwDj/Rnuj0FtWQU1Oqj6J2fCmkVp85PkWJRUCsHBbV6KWpQU0irAZ3zU6RY8ghqeYe1Mga1wbGZeZcgGSpiUFNIqwmd81OkWLIOapD/rJo6P6XoihbUFNJqROf8FCmWgUf69fVnCajzs16KFNQU0mpGnZ8ixaOgVg4KavVRlKCmkFZD6vwUKR4FtXJQUKuPInR+KqTVmDo/RYol66A2PNKX6fbaUVCTosszqAUNaWa2v5ndbGbbzGy9mZ0xznIfN7N7zWyrmT1sZh8PWYd0Tp2fIpGijF/q/CwHBbV6ySuohZ5JuxwYARYAZwJXmNlRbZYz4GxgLnAycK6ZvSVwLdIhdX6KAAUav+ra+bll1/Rca+iWglq95BHUgoU0M5sFnAZc4O5D7n4H8B3grNZl3X2Nu9/j7qPufj/w98DxoWqR7qnzU+qsiOOXOj/LQZ2f9ZJ1UDN3D7MisxcCd7r7jKbrzgNe5e5vmuB+BtwDfNXdv9Lm9lXAKoB58+Yd89kLvhCk3hDmLpzF4IZteZexh15r8hnTGNnXWLbfk0Hq2bVzAdOmbwyyrhBUz8SKVg/AG173wbvd/dg0t5HN+DX/mAsv/VKi+nb37050v8ksmDrAxrHhva4f6B9NZXuT2X/3TDZN2Q7AnGk7c6mh2cDIfgz3P9Px8nOnbk+xGhjeuZCB6RtS3Ua3ilZTVvV0eqDjt77+PT2NXyH3Gp0NbG65bjOwzyT3u5BoRu+adje6+5XAlQBLFx/iN625q7cqAzp19XEUqR4IV9P9q5dw88ov9ryeJ9auZtHyNT2vJxTVM7Gi1ZOh1Mevxc9d5pf+5vHEBQ4vHkl83/F8bPYSLhla3/a2pQc9FXx7kzlj+9FcP/Pnz/580sK1mdfQbNmjK3nw4Fu6uk+aMy3r1p7P0uUXp7b+JIpWU1b1LI3/TXsWNeQ+aUPAnJbr5gBbx7uDmZ1LtG/HG919749zkht1fkrNFH780lef5aCvPusl7a8/Q4a0B4A+Mzus6boVwH3tFjazdwHnA69198cC1iGBqPNTaqQU45c6P8tBQa1e0gxqwUKau28DbgIuMrNZZnY88GbgutZlzexM4LPASe7+UKgaJDx1fkodlGn8qmvnZ9nCmoJavaQV1EIfguMDwAzgSeAG4P3ufp+ZnWBmQ03LfQY4ALjLzIbiy1473UoxqPNTaqI045c6P8tBnZ/1kkZQCxrS3H2Tu69091nuvtjdr4+vv93dZzctd4i7T3P32U2X94WsRcLSOT+l6so4fimolYOCWn2EDmo6LZR0TOf8FCkeBbVyUFCrj5Dn/FRIk66p81OkWBTUykFBrV5CBDWFNElEnZ8ixaLOz3JQUJNuKKRJYur8FCkWdX6Wg4KadEohTXqizk+RzpjDPuvDnIZvIur8LAcFNemEQpr0TJ2fIp3LIqiB9lMrAx2iQyajkCZBqPNTpHMKaukpW1ADzarJ+BTSJCh1fop0RkEtPQpqUhUKaRKcOj9FOlPloJZ3WFNQkypQSJNUNDo/H95xQN6liBRaVYMa5D+rps5PKTuFNEnNohsfpu/JKer8FJnEPutdnZ8pUlCTslJIk1TZjl3q/BTpUFVn1RTUuqfOTwGFNMmAOj9FOqeglp6yBTXQrFrdKaRJZtT5KdKZrILalJFs3wIU1JJRUKuvvrwLKKIfnnkVIzO3T7rc9weB90y+vv7tM3nNN8/pvbAKOGLNem578KXwLjhr7p15lyNNdo+O0d/3V+weHWNK39S8y6m9fdY7W5dYovs+uPvTjLF10uXOfaCz9U2ZMpuDF38yUS2tGkFt6UFPBVlfErdtWM5JC9fmtv0kvr3lRRybdxEFNzY6xrSplzI2OsbUioxhmklro5OAluf6yk7n/CymHYNbmGIPsWNwS96lSCzpjFonAa0bu3cPBV0f5D+rVsbOz8GxmXmXUGhDg0OYPcTQYPjXa14U0iQXOudnseweHWNkaBtmzsjQNnaPjuVdksSy6vzMQ95BDcr39ae++mxvbHSM7Vu3Y+Zs37qdsYqMYQppkhud87M4dgxugUYOcDSbVkAKaukpY1BTWNvT0ODQHmNYVWbTFNIkV+r8zF9jFq2ZZtOKSUEtPWULaqBZtYbGLFqzqsymKaRJIajzMz97zKI1aDatsBTU0qOgVk57zKI1VGQ2TSFNCkPn/Mxeu1m0Bs2mFVdVg9rwSF/uYU1BrVzazaI1VGE2LWhIM7P9zexmM9tmZuvN7IxxljMz+7yZPR1f1phZsl5zqRR1fmar7SxaQ81m08o2flU1qEH+s2pl7Pysa1BrO4vWUIHZtNAzaZcDI8AC4EzgCjM7qs1yq4CVwArgBcAfAu8NXIuUlDo/szHRLFpDzWbTSjd+Zdn5WcczFGzZNT3vErpSt6A20SxaQ9ln04KFNDObBZwGXODuQ+5+B/Ad4Kw2i78duMTdH3P3x4FLgHeEqkXKT52f6ZtwFq2hJrNpZR+/FNTSU8YZtbqEtQln0RpKPptm7mH+uM3shcCd7j6j6brzgFe5+5talt0M/IG7/3v887HAj9x9nzbrXUX0yZV58+Yd89kLvhCk3omsGjw7+DqvnPv14OtsZ+7CWQxumHh2JEsh6hle2M+y/Z4MUs+unQuYNn1jkHWFkF89m5k+7SLMdk26pPs0du76NDAn/bLaeMPrPni3u6d6sPWsxq+L/uJLKT4K2N0ffet67gN/HHzdXzr875q2szv4+gEWTB1g49jwXtcP9I+msr3J7L97JpumRDM1c6btzKWGZgMj+zHc/0zHy8+dmv6B1Id3LmRg+obUt7O3zfT3/VnHY9jI6KfIYww7+Q8+1NP4FfK0ULOBzS3XbQb2GrjaLLsZmG1m5i2p0d2vBK4EWLr4EL9pzV3hKh5PB6d66lYmdQOnrj4us211IlQ9969ewqmv+mnPp5J6Yu1qFi1f03M9oeRVz7anBhnZOvngBmC2i30P+Byz5s9NuapcpT5+LVnyXL/q9sfDVTyOpKeSmsylv9mz9uHFI8G38bHZS7hkaH3b2/I4jdQZ24/m+pk/f/bnvE8ltezRlTx48C1d3ef0OfekVE1k3drzWbr84lS30c7mpzazfUvnY9jcAz7HvvP3Tbmq8ELukzbE3jF1DrQ9P0nrsnOAodYBTqRBnZ/hdLIvWqsa7JuWyfi174N7zxKFVuWvPvP++rNsX31CNfdT62RftFZl3TctZEh7AOgzs8OarlsB3Ndm2fvi2yZbTuRZ6vwMo6N90VpVf9+0zMavfR8cziSsZSHroAb576emzs/8dbQvWquS7psWLKS5+zbgJuAiM5tlZscDbwaua7P414GPmtmBZrYI+BhwbahapLrU+dmbJLNoDVWeTctj/KpSUFNDQfFVJaglmUVrKONsWuhDcHwAmAE8CdwAvN/d7zOzE8ysOcJ+FbgV+BVwL/Dd+DqRSanzM7lEs2gN1Z9Ny3z8qkpQA3V+lkEVOj8TzaI1lHA2LWhIc/dN7r7S3We5+2J3vz6+/nZ3n920nLv7anffP76s1v5o0g2d87N7vcyiNVR8Ni2X8UtBLTkFtWTKGtR6mUVrKNtsmk4LJaWmc352rqdZtIbqz6blQkEtOQW1ZMoY1HqaRWso2WyaQpqUnjo/JxdiFq2hyrNpeVJQS06dn8mUKaiFmEVrKNNsmkKaVII6PycWZBatQbNpqVHnZ2+KENTKFtbKEtSCzKI1lGg2TSGtjf7tMwu9PmlPnZ/jGxsOe+DR0OuTPfUa1PrG2h2DN7mpbY/pOzl1fpZDGYLayM6wY07o9aUl5BkHKuM13zyno+WKdnR/aXR+Hsgpb/gwN6/8Yt7lFMacgxZ0tFzRzshQZ/s+OMzmZQOJ7rviiQs7Wu6cEw7ki488kWgb3Rh4pD+VMxSMZ91j83M5Q0Gz2zYsz/0MBd1oBLW0z1CQ1PyDOwvfeZ0BIS2aSZPKUeenVIXOUJCcZtSSKcOsWp0opEllqfNTqkBBLTkFtWQU1IpDIU0qTZ2fUgUKasmp8zMZBbViUEiTylPnp1RBFp2fVQ1qkP+smjo/JQmFNKmFRufnwzsOyLsUkZ5kEdSyCGvq/CwHBbV8KaRJbcy493H6npyihgIpPX39mZyCWveqcM7PslJIk1qxHbvU+SmVoKCWnIJaMgpq2VNIk1pS56dUgYJacgpqySioZUshTWpLnZ9SBVUKalNGsn1LUudnMgpq2VFIk1pT56dUgTo/e1OEoFa2sKaglg2FNKk9nfNTsmCefshR52dyeQc1KN+smoJa+hTSRGic83OWGgokVf1rH6N/7WOpbqNKX38qqBXf4NhMhbUUKaSJxHTOT8mKglrnFNTKQUEtHQppIi3U+SlZUFDrnIJaOSiohaeQJtKGOj8lCwpqncsjqA2P9GW6zVYKaqKQJjIOdX5KFrIIaur8TC7vWTV1ftZbkJBmZvub2c1mts3M1pvZGRMs+3Ezu9fMtprZw2b28RA1iKRBnZ/1kPcYlnZQA3V+9iLvoAblm1VTUAsj1Eza5cAIsAA4E7jCzI4aZ1kDzgbmAicD55rZWwLVIRKcOj9rIfcxTJ2f3aljUNuya3reJXRF5/zsXc8hzcxmAacBF7j7kLvfAXwHOKvd8u6+xt3vcfdRd78f+Hvg+F7rEEmTOj+rq2hjmIJa5+oY1Mo2owaaVeuFeY8HWDSzFwJ3uvuMpuvOA17l7m+a5L4G3AN81d2/Ms4yq4BV8Y/PB+7tqeCw5gG/zbuIFkWrSfVMTPVM7gh33yetlac5hhV8/ILi/b5Vz8SKVg8Ur6ai1dPT+BWidWU2sLnlus1AJ0VdSDSbd814C7j7lcCVAGb2M3c/NlmZ4RWtHiheTapnYqpncmb2s5Q3kdoYVuTxC4pXk+qZWNHqgeLVVMR6ern/pF93mtmPzczHudwBDAFzWu42B9g6yXrPJdqv443unv78u4jUksYwESmrSWfS3P3EiW6P9+foM7PD3P3X8dUrgPsmuM+7gPOBV7p7+m1NIlJbGsNEpKx6bhxw923ATcBFZjbLzI4H3gxc1255MzsT+Cxwkrs/1OXmruyp2PCKVg8UrybVMzHVM7lUa8pwDKvdc5uA6plY0eqB4tVUqXp6bhyA6BhDwNXAScDTwPnufn182wnA99x9dvzzw8BBQPPXA99w9/f1XIiISAIaw0SkiIKENBEREREJS6eFEhERESkghTQRERGRAip0SMv7fHrd1GCRz5vZ0/FlTXygy6C6qCeTc6R28zuKl+83s7VmlkpHXJevmReZ2b+a2ZCZbTSzD+VVj5kNmNlX4jo2mdmtZnZgCvWca2Y/M7NhM7t2kmU/YmYbzGyzmV1tZgOh6+mmJjN7u5ndbWZbzOyx+G8sxLEeU6Hxq6d6MjvHs8awMPXUdQxLe/wqdEijAOfT66KGVcBKotb9FwB/CLw3wPaT1pPVOVK7+R0BfBx4MoU6uqrHzOYB3we+ChwAHAr8U171AB8CXkb02lkEPANclkI9TwCfIdpJflxm9jqiQ0y8FlgKPBf40xTq6bgmYCbwYaIjir8kru28lGoKQeNX8nqyPMezxrAA9VDfMSzd8cvdC3kBZhG9MA5vuu464OIO738pcFlWNQB3Aquafj4H+ElRnpMQz0ev9QCHAP8JvB54LM/XDNEhFK4LXUMP9VwBrGn6+Y3A/SnW9hng2gluvx74bNPPrwU2pPx8TVhTm+U/CtyaZk1Z/O7Hub/Gr8DPR4iaNIZpDEtaT5vlOxq/ijyTdjgw5u4PNF33C2CiTzjAs+fTO4EJDkaZQg1HxbdNtlxW9Twr4PPRaz2XAZ8AdgSuI0k9LwU2mdmdZvZkPDW/OMd6rgKON7NFZjaT6BPr9wLX0412r+cFZnZATvW080rCv6ZD0fjVWz3PSnH8SlKTxjCNYaF0NH4VOaSlek7QFGpoXXYzMDvwfh1Jn5MLCfN8JK7HzE4B+tz95sA1JKqH6DhXbyeaol8MPAzckGM9DwCPAI8DW4AjgYsC19ONdq9n6OzvL3Vm9k7gWOAv8q5lHBq/equn2YWkM351VZPGsEnr0RjWoW7Gr9xCmpXjfHrd1NC67BxgyON5zUC6fk4CPx+J6rHotDtrgP8n8PYT1RPbAdzs7ne5+06ifRVebmb75lTPFcB0on1LZhEdAT/PT6HtXs8wyd9fFsxsJXAx8Hp3/21ONWj8SrceIJPzo2oMC1ePxrAOdDt+5RbS3P1Ed7dxLq8gSuV9ZnZY0906PZ/eaz3M+fS6qeG++LaOas2gnjSej6T1HEa04+btZraB6I/39+Kum6U51APwS6D5Dajx/5AzB93Us4Jof4ZN8ZvRZcCL452D89Du9bzR3Z/OqR4AzOxk4K+BN7n7r/KqQ+NX6vVkMX51U5PGsMnr0Rg2iUTjV1o70QXaEe9viaZvZwHHE01XHjXOsmcCG4Aj86gBeB/RDqUHEnW23Ae8L6/nJK3nI0k9QB+wsOlyKlFHzEJgak7Pz2uAQeBoYBrwl8DtOf6+rgFuBPaN6/kE8HgK9fQRfdr9HNEOwNOJvsJpXe7k+PXzPKIOux/S4U7vKdb0GqJTNr0yzdd01r/7eFmNXxk8H0lr0himMSxAPYnGr1Rf/AEe/P7ALcA2ou+6z2i67QSi6fjGzw8Du4imOBuXr6RVQ5vtG9F0+Kb4sob4tFtZPCdZPR9J62m5z4mk0BnVbT3A+4n2nxgEbgUOzvH3dQDwTaLW/meAO4AXp1DPhUSfuJsvFxLt0zIELG5a9qPARqL9S64BBlL6nXVUE/AjYLTlNf29NGpK83c/zu9f41cO41c3NbXc50Q0hmkM66IeEo5fOneniIiISAEVubtTREREpLYU0kREREQKSCFNREREpIAU0kREREQKSCFNREREpIAU0kREREQKSCFNREREpIAU0kREREQK6P8HwdgJbcJzZ/IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff941591438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: heaviside\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FNN for MNIST\n",
    "# mnist数据集上运用FNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Estimator API (formerly `tf.contrib.learn`)\n",
    "## 使用Estimator API（以前的`tf.contrib.learn`）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zhangxj/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: `tf.examples.tutorials.mnist` is deprecated. We will use `tf.keras.datasets.mnist` instead. Moreover, the `tf.contrib.learn` API was promoted to `tf.estimators` and `tf.feature_columns`, and it has changed considerably. In particular, there is no `infer_real_valued_columns_from_input()` function or `SKCompat` class.\n",
    "\n",
    "**警告**：`tf.examples.tutorials.mnist`不可用了，我们使用`tf.keras.datasets.mnist`替代。而且，`tf.contrib.learn`API升级为`tf.estimators`和`tf.feature_columns`，而且有了很大的改变。尤其是，没有了`infer_real_valued_columns_from_input()`函数和`SKCompat`类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0 # 三维矩阵重塑为二维矩阵，列数为28×28=784; /255.0对图像归一化处理，使其范围为[0, 1]\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32) # uint8转int32\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:] # \n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpef5o9cas\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpef5o9cas', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc43eadd080>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpef5o9cas/model.ckpt.\n",
      "INFO:tensorflow:loss = 111.780815, step = 1\n",
      "INFO:tensorflow:global_step/sec: 250.957\n",
      "INFO:tensorflow:loss = 12.076608, step = 101 (0.399 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.476\n",
      "INFO:tensorflow:loss = 13.555847, step = 201 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.005\n",
      "INFO:tensorflow:loss = 5.7416573, step = 301 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.477\n",
      "INFO:tensorflow:loss = 5.3654323, step = 401 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.377\n",
      "INFO:tensorflow:loss = 12.688744, step = 501 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.586\n",
      "INFO:tensorflow:loss = 9.025182, step = 601 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.71\n",
      "INFO:tensorflow:loss = 2.8021545, step = 701 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 302.635\n",
      "INFO:tensorflow:loss = 2.2092848, step = 801 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.197\n",
      "INFO:tensorflow:loss = 16.961315, step = 901 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.335\n",
      "INFO:tensorflow:loss = 19.833868, step = 1001 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.153\n",
      "INFO:tensorflow:loss = 3.6737537, step = 1101 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.301\n",
      "INFO:tensorflow:loss = 4.20737, step = 1201 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 297.195\n",
      "INFO:tensorflow:loss = 10.750921, step = 1301 (0.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.282\n",
      "INFO:tensorflow:loss = 4.932289, step = 1401 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.584\n",
      "INFO:tensorflow:loss = 1.0766221, step = 1501 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.643\n",
      "INFO:tensorflow:loss = 4.104335, step = 1601 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.603\n",
      "INFO:tensorflow:loss = 5.69323, step = 1701 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.439\n",
      "INFO:tensorflow:loss = 3.3961508, step = 1801 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 290.29\n",
      "INFO:tensorflow:loss = 1.152564, step = 1901 (0.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.92\n",
      "INFO:tensorflow:loss = 6.0209775, step = 2001 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 299.786\n",
      "INFO:tensorflow:loss = 1.6482629, step = 2101 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 283.991\n",
      "INFO:tensorflow:loss = 1.057581, step = 2201 (0.352 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.52\n",
      "INFO:tensorflow:loss = 1.5657434, step = 2301 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.496\n",
      "INFO:tensorflow:loss = 2.3812504, step = 2401 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 297.514\n",
      "INFO:tensorflow:loss = 1.6704656, step = 2501 (0.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.778\n",
      "INFO:tensorflow:loss = 7.8489, step = 2601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.606\n",
      "INFO:tensorflow:loss = 0.6591275, step = 2701 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.061\n",
      "INFO:tensorflow:loss = 2.5672238, step = 2801 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.237\n",
      "INFO:tensorflow:loss = 13.744191, step = 2901 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.798\n",
      "INFO:tensorflow:loss = 2.2410645, step = 3001 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.052\n",
      "INFO:tensorflow:loss = 4.3045053, step = 3101 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.477\n",
      "INFO:tensorflow:loss = 2.2755609, step = 3201 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.479\n",
      "INFO:tensorflow:loss = 1.0581148, step = 3301 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 263.322\n",
      "INFO:tensorflow:loss = 4.0066447, step = 3401 (0.380 sec)\n",
      "INFO:tensorflow:global_step/sec: 268.567\n",
      "INFO:tensorflow:loss = 4.954536, step = 3501 (0.373 sec)\n",
      "INFO:tensorflow:global_step/sec: 282.206\n",
      "INFO:tensorflow:loss = 5.630864, step = 3601 (0.354 sec)\n",
      "INFO:tensorflow:global_step/sec: 249.534\n",
      "INFO:tensorflow:loss = 2.3640208, step = 3701 (0.401 sec)\n",
      "INFO:tensorflow:global_step/sec: 283.524\n",
      "INFO:tensorflow:loss = 0.84995127, step = 3801 (0.353 sec)\n",
      "INFO:tensorflow:global_step/sec: 258.3\n",
      "INFO:tensorflow:loss = 0.3012246, step = 3901 (0.388 sec)\n",
      "INFO:tensorflow:global_step/sec: 246.997\n",
      "INFO:tensorflow:loss = 6.577649, step = 4001 (0.404 sec)\n",
      "INFO:tensorflow:global_step/sec: 265.485\n",
      "INFO:tensorflow:loss = 2.971659, step = 4101 (0.377 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.395\n",
      "INFO:tensorflow:loss = 0.5645688, step = 4201 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.394\n",
      "INFO:tensorflow:loss = 7.1532774, step = 4301 (0.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.213\n",
      "INFO:tensorflow:loss = 3.5584726, step = 4401 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 265.612\n",
      "INFO:tensorflow:loss = 1.017456, step = 4501 (0.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 295.755\n",
      "INFO:tensorflow:loss = 1.1326538, step = 4601 (0.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.251\n",
      "INFO:tensorflow:loss = 1.285548, step = 4701 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.611\n",
      "INFO:tensorflow:loss = 0.38332024, step = 4801 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 286.498\n",
      "INFO:tensorflow:loss = 2.91071, step = 4901 (0.352 sec)\n",
      "INFO:tensorflow:global_step/sec: 278.946\n",
      "INFO:tensorflow:loss = 1.4282216, step = 5001 (0.355 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.671\n",
      "INFO:tensorflow:loss = 1.6251184, step = 5101 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 284.725\n",
      "INFO:tensorflow:loss = 2.639079, step = 5201 (0.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 250.276\n",
      "INFO:tensorflow:loss = 0.4484363, step = 5301 (0.400 sec)\n",
      "INFO:tensorflow:global_step/sec: 298.719\n",
      "INFO:tensorflow:loss = 0.39229402, step = 5401 (0.335 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.955\n",
      "INFO:tensorflow:loss = 3.3378286, step = 5501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 243.673\n",
      "INFO:tensorflow:loss = 0.69804645, step = 5601 (0.410 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.726\n",
      "INFO:tensorflow:loss = 6.797665, step = 5701 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 284.163\n",
      "INFO:tensorflow:loss = 0.39831948, step = 5801 (0.352 sec)\n",
      "INFO:tensorflow:global_step/sec: 251.209\n",
      "INFO:tensorflow:loss = 0.62915164, step = 5901 (0.398 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.724\n",
      "INFO:tensorflow:loss = 0.2069949, step = 6001 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 289.371\n",
      "INFO:tensorflow:loss = 2.2122707, step = 6101 (0.348 sec)\n",
      "INFO:tensorflow:global_step/sec: 227.549\n",
      "INFO:tensorflow:loss = 0.8599986, step = 6201 (0.437 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.133\n",
      "INFO:tensorflow:loss = 1.6847079, step = 6301 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.574\n",
      "INFO:tensorflow:loss = 0.96230966, step = 6401 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 299.945\n",
      "INFO:tensorflow:loss = 0.97663426, step = 6501 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 251.859\n",
      "INFO:tensorflow:loss = 0.18583757, step = 6601 (0.397 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.82\n",
      "INFO:tensorflow:loss = 1.2266096, step = 6701 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.757\n",
      "INFO:tensorflow:loss = 14.724922, step = 6801 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.317\n",
      "INFO:tensorflow:loss = 0.14505899, step = 6901 (0.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 235.806\n",
      "INFO:tensorflow:loss = 0.71012145, step = 7001 (0.423 sec)\n",
      "INFO:tensorflow:global_step/sec: 288.182\n",
      "INFO:tensorflow:loss = 0.33212817, step = 7101 (0.347 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.359\n",
      "INFO:tensorflow:loss = 0.5143834, step = 7201 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.641\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 1.0918583, step = 7301 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.08\n",
      "INFO:tensorflow:loss = 1.0798061, step = 7401 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.28\n",
      "INFO:tensorflow:loss = 0.2744903, step = 7501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.916\n",
      "INFO:tensorflow:loss = 0.2125386, step = 7601 (0.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.009\n",
      "INFO:tensorflow:loss = 0.26754993, step = 7701 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.833\n",
      "INFO:tensorflow:loss = 0.18102524, step = 7801 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.517\n",
      "INFO:tensorflow:loss = 0.23630363, step = 7901 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.148\n",
      "INFO:tensorflow:loss = 1.5450534, step = 8001 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.641\n",
      "INFO:tensorflow:loss = 0.42742676, step = 8101 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.108\n",
      "INFO:tensorflow:loss = 1.0946605, step = 8201 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.645\n",
      "INFO:tensorflow:loss = 0.64679384, step = 8301 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.381\n",
      "INFO:tensorflow:loss = 0.71875054, step = 8401 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.788\n",
      "INFO:tensorflow:loss = 0.68842673, step = 8501 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.662\n",
      "INFO:tensorflow:loss = 0.37234426, step = 8601 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.127\n",
      "INFO:tensorflow:loss = 0.17023301, step = 8701 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.775\n",
      "INFO:tensorflow:loss = 0.24427879, step = 8801 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.149\n",
      "INFO:tensorflow:loss = 0.86516225, step = 8901 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 287.187\n",
      "INFO:tensorflow:loss = 0.9031214, step = 9001 (0.350 sec)\n",
      "INFO:tensorflow:global_step/sec: 287.867\n",
      "INFO:tensorflow:loss = 3.52858, step = 9101 (0.346 sec)\n",
      "INFO:tensorflow:global_step/sec: 276.265\n",
      "INFO:tensorflow:loss = 0.30415815, step = 9201 (0.362 sec)\n",
      "INFO:tensorflow:global_step/sec: 273.617\n",
      "INFO:tensorflow:loss = 0.3778218, step = 9301 (0.366 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.262\n",
      "INFO:tensorflow:loss = 0.16113636, step = 9401 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.356\n",
      "INFO:tensorflow:loss = 0.1206472, step = 9501 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 252.208\n",
      "INFO:tensorflow:loss = 0.13819149, step = 9601 (0.396 sec)\n",
      "INFO:tensorflow:global_step/sec: 274.384\n",
      "INFO:tensorflow:loss = 0.99463797, step = 9701 (0.364 sec)\n",
      "INFO:tensorflow:global_step/sec: 284.559\n",
      "INFO:tensorflow:loss = 0.3311089, step = 9801 (0.355 sec)\n",
      "INFO:tensorflow:global_step/sec: 241.394\n",
      "INFO:tensorflow:loss = 3.5283735, step = 9901 (0.411 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.143\n",
      "INFO:tensorflow:loss = 0.16843192, step = 10001 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.594\n",
      "INFO:tensorflow:loss = 0.38727516, step = 10101 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.804\n",
      "INFO:tensorflow:loss = 0.22705327, step = 10201 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.255\n",
      "INFO:tensorflow:loss = 0.06325632, step = 10301 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.036\n",
      "INFO:tensorflow:loss = 0.15008864, step = 10401 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 289.978\n",
      "INFO:tensorflow:loss = 0.31108052, step = 10501 (0.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.697\n",
      "INFO:tensorflow:loss = 0.17205164, step = 10601 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.608\n",
      "INFO:tensorflow:loss = 0.10688186, step = 10701 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.864\n",
      "INFO:tensorflow:loss = 1.0923142, step = 10801 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.059\n",
      "INFO:tensorflow:loss = 0.015977299, step = 10901 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.119\n",
      "INFO:tensorflow:loss = 0.13953415, step = 11001 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.914\n",
      "INFO:tensorflow:loss = 0.21765679, step = 11101 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.821\n",
      "INFO:tensorflow:loss = 0.35784346, step = 11201 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 302.306\n",
      "INFO:tensorflow:loss = 0.12759447, step = 11301 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.185\n",
      "INFO:tensorflow:loss = 0.48386928, step = 11401 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.24\n",
      "INFO:tensorflow:loss = 0.5787152, step = 11501 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.961\n",
      "INFO:tensorflow:loss = 0.4042919, step = 11601 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.308\n",
      "INFO:tensorflow:loss = 0.09635724, step = 11701 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.665\n",
      "INFO:tensorflow:loss = 0.18058285, step = 11801 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.049\n",
      "INFO:tensorflow:loss = 0.8052333, step = 11901 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.365\n",
      "INFO:tensorflow:loss = 0.0512689, step = 12001 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.175\n",
      "INFO:tensorflow:loss = 0.090965, step = 12101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.617\n",
      "INFO:tensorflow:loss = 0.2105716, step = 12201 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.024\n",
      "INFO:tensorflow:loss = 0.012889499, step = 12301 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.903\n",
      "INFO:tensorflow:loss = 0.033778343, step = 12401 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.225\n",
      "INFO:tensorflow:loss = 0.34765586, step = 12501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.957\n",
      "INFO:tensorflow:loss = 0.3495779, step = 12601 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.072\n",
      "INFO:tensorflow:loss = 0.11323053, step = 12701 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.659\n",
      "INFO:tensorflow:loss = 0.5477688, step = 12801 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.834\n",
      "INFO:tensorflow:loss = 0.34136483, step = 12901 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.728\n",
      "INFO:tensorflow:loss = 0.17293607, step = 13001 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.69\n",
      "INFO:tensorflow:loss = 0.40136817, step = 13101 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.343\n",
      "INFO:tensorflow:loss = 0.35284895, step = 13201 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.472\n",
      "INFO:tensorflow:loss = 0.23284116, step = 13301 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.433\n",
      "INFO:tensorflow:loss = 1.1244767, step = 13401 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.239\n",
      "INFO:tensorflow:loss = 0.05523292, step = 13501 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.436\n",
      "INFO:tensorflow:loss = 0.11063747, step = 13601 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.701\n",
      "INFO:tensorflow:loss = 0.063106805, step = 13701 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.793\n",
      "INFO:tensorflow:loss = 0.31656402, step = 13801 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.637\n",
      "INFO:tensorflow:loss = 0.047893934, step = 13901 (0.329 sec)\n",
      "INFO:tensorflow:global_step/sec: 274.921\n",
      "INFO:tensorflow:loss = 0.16110852, step = 14001 (0.364 sec)\n",
      "INFO:tensorflow:global_step/sec: 224.054\n",
      "INFO:tensorflow:loss = 0.01551877, step = 14101 (0.446 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.187\n",
      "INFO:tensorflow:loss = 0.2367358, step = 14201 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.256\n",
      "INFO:tensorflow:loss = 0.46101433, step = 14301 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.959\n",
      "INFO:tensorflow:loss = 0.1918748, step = 14401 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.646\n",
      "INFO:tensorflow:loss = 0.10151374, step = 14501 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 319.17\n",
      "INFO:tensorflow:loss = 0.025223596, step = 14601 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.254\n",
      "INFO:tensorflow:loss = 0.33509168, step = 14701 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.778\n",
      "INFO:tensorflow:loss = 0.046682555, step = 14801 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.602\n",
      "INFO:tensorflow:loss = 0.13630456, step = 14901 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.613\n",
      "INFO:tensorflow:loss = 0.149914, step = 15001 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.065\n",
      "INFO:tensorflow:loss = 0.0618653, step = 15101 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.108\n",
      "INFO:tensorflow:loss = 0.11740936, step = 15201 (0.329 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.906\n",
      "INFO:tensorflow:loss = 0.047532067, step = 15301 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.718\n",
      "INFO:tensorflow:loss = 0.07267468, step = 15401 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.238\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.126877, step = 15501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.731\n",
      "INFO:tensorflow:loss = 0.2519959, step = 15601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.729\n",
      "INFO:tensorflow:loss = 0.011158925, step = 15701 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.244\n",
      "INFO:tensorflow:loss = 0.114383355, step = 15801 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.732\n",
      "INFO:tensorflow:loss = 0.16539742, step = 15901 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 302.57\n",
      "INFO:tensorflow:loss = 0.029493548, step = 16001 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.997\n",
      "INFO:tensorflow:loss = 0.07467363, step = 16101 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.619\n",
      "INFO:tensorflow:loss = 0.0994162, step = 16201 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.83\n",
      "INFO:tensorflow:loss = 0.25762385, step = 16301 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.062\n",
      "INFO:tensorflow:loss = 0.1237788, step = 16401 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.626\n",
      "INFO:tensorflow:loss = 0.013188369, step = 16501 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.079\n",
      "INFO:tensorflow:loss = 0.08598665, step = 16601 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.298\n",
      "INFO:tensorflow:loss = 0.054134797, step = 16701 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.07\n",
      "INFO:tensorflow:loss = 0.5903826, step = 16801 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.76\n",
      "INFO:tensorflow:loss = 0.050475527, step = 16901 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.721\n",
      "INFO:tensorflow:loss = 0.060102403, step = 17001 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.484\n",
      "INFO:tensorflow:loss = 0.41239104, step = 17101 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.566\n",
      "INFO:tensorflow:loss = 0.15150332, step = 17201 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.658\n",
      "INFO:tensorflow:loss = 0.14251308, step = 17301 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.685\n",
      "INFO:tensorflow:loss = 0.07866496, step = 17401 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.761\n",
      "INFO:tensorflow:loss = 0.03251211, step = 17501 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.27\n",
      "INFO:tensorflow:loss = 0.23213261, step = 17601 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.761\n",
      "INFO:tensorflow:loss = 0.11661323, step = 17701 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.8\n",
      "INFO:tensorflow:loss = 0.099479824, step = 17801 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.217\n",
      "INFO:tensorflow:loss = 0.04467143, step = 17901 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.606\n",
      "INFO:tensorflow:loss = 0.090608284, step = 18001 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.51\n",
      "INFO:tensorflow:loss = 0.1329665, step = 18101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.153\n",
      "INFO:tensorflow:loss = 0.41236582, step = 18201 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.666\n",
      "INFO:tensorflow:loss = 0.090678535, step = 18301 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.652\n",
      "INFO:tensorflow:loss = 0.056909263, step = 18401 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.06\n",
      "INFO:tensorflow:loss = 0.092499234, step = 18501 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.932\n",
      "INFO:tensorflow:loss = 0.03253399, step = 18601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.641\n",
      "INFO:tensorflow:loss = 0.021963097, step = 18701 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 296.024\n",
      "INFO:tensorflow:loss = 0.116077326, step = 18801 (0.338 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.068\n",
      "INFO:tensorflow:loss = 0.06178749, step = 18901 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.256\n",
      "INFO:tensorflow:loss = 0.09194968, step = 19001 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.885\n",
      "INFO:tensorflow:loss = 0.05742636, step = 19101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 324.418\n",
      "INFO:tensorflow:loss = 0.010413496, step = 19201 (0.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.288\n",
      "INFO:tensorflow:loss = 0.02876101, step = 19301 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.4\n",
      "INFO:tensorflow:loss = 0.468392, step = 19401 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.039\n",
      "INFO:tensorflow:loss = 0.16739881, step = 19501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.562\n",
      "INFO:tensorflow:loss = 0.2099161, step = 19601 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.464\n",
      "INFO:tensorflow:loss = 0.027355721, step = 19701 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.566\n",
      "INFO:tensorflow:loss = 0.0627464, step = 19801 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.929\n",
      "INFO:tensorflow:loss = 0.03507179, step = 19901 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.676\n",
      "INFO:tensorflow:loss = 0.11070241, step = 20001 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.018\n",
      "INFO:tensorflow:loss = 0.03444156, step = 20101 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.426\n",
      "INFO:tensorflow:loss = 0.035964526, step = 20201 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.276\n",
      "INFO:tensorflow:loss = 0.09771008, step = 20301 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.748\n",
      "INFO:tensorflow:loss = 0.016143048, step = 20401 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.46\n",
      "INFO:tensorflow:loss = 0.0822144, step = 20501 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.921\n",
      "INFO:tensorflow:loss = 0.062410217, step = 20601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.195\n",
      "INFO:tensorflow:loss = 0.07366611, step = 20701 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.56\n",
      "INFO:tensorflow:loss = 0.08461101, step = 20801 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.874\n",
      "INFO:tensorflow:loss = 0.020423118, step = 20901 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.75\n",
      "INFO:tensorflow:loss = 0.03244738, step = 21001 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.212\n",
      "INFO:tensorflow:loss = 0.031498864, step = 21101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.644\n",
      "INFO:tensorflow:loss = 0.12966204, step = 21201 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.186\n",
      "INFO:tensorflow:loss = 0.079381056, step = 21301 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.366\n",
      "INFO:tensorflow:loss = 0.024593541, step = 21401 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.681\n",
      "INFO:tensorflow:loss = 0.10983623, step = 21501 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.748\n",
      "INFO:tensorflow:loss = 0.1891157, step = 21601 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.22\n",
      "INFO:tensorflow:loss = 0.02556461, step = 21701 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.391\n",
      "INFO:tensorflow:loss = 0.009164347, step = 21801 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.716\n",
      "INFO:tensorflow:loss = 0.004706936, step = 21901 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.975\n",
      "INFO:tensorflow:loss = 0.11767615, step = 22001 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.035\n",
      "INFO:tensorflow:loss = 0.020255007, step = 22101 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.645\n",
      "INFO:tensorflow:loss = 0.04092497, step = 22201 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.466\n",
      "INFO:tensorflow:loss = 0.090483174, step = 22301 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.187\n",
      "INFO:tensorflow:loss = 0.05669486, step = 22401 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.143\n",
      "INFO:tensorflow:loss = 0.037943147, step = 22501 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.04\n",
      "INFO:tensorflow:loss = 0.07792105, step = 22601 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.801\n",
      "INFO:tensorflow:loss = 0.019841043, step = 22701 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.976\n",
      "INFO:tensorflow:loss = 0.088304505, step = 22801 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.668\n",
      "INFO:tensorflow:loss = 0.034965217, step = 22901 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.695\n",
      "INFO:tensorflow:loss = 0.01762864, step = 23001 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.615\n",
      "INFO:tensorflow:loss = 0.06636348, step = 23101 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.067\n",
      "INFO:tensorflow:loss = 0.09917714, step = 23201 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.647\n",
      "INFO:tensorflow:loss = 0.02723683, step = 23301 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.193\n",
      "INFO:tensorflow:loss = 0.07908282, step = 23401 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.505\n",
      "INFO:tensorflow:loss = 0.03783269, step = 23501 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.486\n",
      "INFO:tensorflow:loss = 0.022887085, step = 23601 (0.315 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 316.349\n",
      "INFO:tensorflow:loss = 0.0027646052, step = 23701 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.843\n",
      "INFO:tensorflow:loss = 0.02114948, step = 23801 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.11\n",
      "INFO:tensorflow:loss = 0.079031065, step = 23901 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.146\n",
      "INFO:tensorflow:loss = 0.05990723, step = 24001 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.62\n",
      "INFO:tensorflow:loss = 0.028723562, step = 24101 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.221\n",
      "INFO:tensorflow:loss = 0.011850528, step = 24201 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.921\n",
      "INFO:tensorflow:loss = 0.033385336, step = 24301 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.588\n",
      "INFO:tensorflow:loss = 0.044025645, step = 24401 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.7\n",
      "INFO:tensorflow:loss = 0.19095513, step = 24501 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.108\n",
      "INFO:tensorflow:loss = 0.046910904, step = 24601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.198\n",
      "INFO:tensorflow:loss = 0.012713869, step = 24701 (0.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.055\n",
      "INFO:tensorflow:loss = 0.02150248, step = 24801 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.103\n",
      "INFO:tensorflow:loss = 0.0059223827, step = 24901 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.248\n",
      "INFO:tensorflow:loss = 0.03873863, step = 25001 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.842\n",
      "INFO:tensorflow:loss = 0.014573428, step = 25101 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.674\n",
      "INFO:tensorflow:loss = 0.007885657, step = 25201 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.041\n",
      "INFO:tensorflow:loss = 0.010476458, step = 25301 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.48\n",
      "INFO:tensorflow:loss = 0.0051848534, step = 25401 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.042\n",
      "INFO:tensorflow:loss = 0.04905213, step = 25501 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.83\n",
      "INFO:tensorflow:loss = 0.059387896, step = 25601 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.389\n",
      "INFO:tensorflow:loss = 0.029962214, step = 25701 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.436\n",
      "INFO:tensorflow:loss = 0.056081478, step = 25801 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.133\n",
      "INFO:tensorflow:loss = 0.099225804, step = 25901 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.2\n",
      "INFO:tensorflow:loss = 0.039235726, step = 26001 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.126\n",
      "INFO:tensorflow:loss = 0.092579, step = 26101 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.523\n",
      "INFO:tensorflow:loss = 0.03051874, step = 26201 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.732\n",
      "INFO:tensorflow:loss = 0.036255576, step = 26301 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.001\n",
      "INFO:tensorflow:loss = 0.10766254, step = 26401 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.923\n",
      "INFO:tensorflow:loss = 0.08628336, step = 26501 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.734\n",
      "INFO:tensorflow:loss = 0.007917318, step = 26601 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 319.499\n",
      "INFO:tensorflow:loss = 0.03151119, step = 26701 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.678\n",
      "INFO:tensorflow:loss = 0.056030095, step = 26801 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.934\n",
      "INFO:tensorflow:loss = 0.21110576, step = 26901 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.871\n",
      "INFO:tensorflow:loss = 0.014680127, step = 27001 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.693\n",
      "INFO:tensorflow:loss = 0.045827262, step = 27101 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.249\n",
      "INFO:tensorflow:loss = 0.10704558, step = 27201 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.182\n",
      "INFO:tensorflow:loss = 0.034066163, step = 27301 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.675\n",
      "INFO:tensorflow:loss = 0.067604184, step = 27401 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.809\n",
      "INFO:tensorflow:loss = 0.01286379, step = 27501 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.4\n",
      "INFO:tensorflow:loss = 0.1337044, step = 27601 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.34\n",
      "INFO:tensorflow:loss = 0.07346885, step = 27701 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.311\n",
      "INFO:tensorflow:loss = 0.0098829055, step = 27801 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.178\n",
      "INFO:tensorflow:loss = 0.047330216, step = 27901 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.706\n",
      "INFO:tensorflow:loss = 0.07171919, step = 28001 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.191\n",
      "INFO:tensorflow:loss = 0.00067206885, step = 28101 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.271\n",
      "INFO:tensorflow:loss = 0.07552392, step = 28201 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.138\n",
      "INFO:tensorflow:loss = 0.0019465036, step = 28301 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.196\n",
      "INFO:tensorflow:loss = 0.008502813, step = 28401 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.192\n",
      "INFO:tensorflow:loss = 0.009259484, step = 28501 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.505\n",
      "INFO:tensorflow:loss = 0.017532835, step = 28601 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.662\n",
      "INFO:tensorflow:loss = 0.017493175, step = 28701 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.518\n",
      "INFO:tensorflow:loss = 0.07082109, step = 28801 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.574\n",
      "INFO:tensorflow:loss = 0.052575953, step = 28901 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.017\n",
      "INFO:tensorflow:loss = 0.0026218495, step = 29001 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.144\n",
      "INFO:tensorflow:loss = 0.01919686, step = 29101 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 271.307\n",
      "INFO:tensorflow:loss = 0.038905166, step = 29201 (0.369 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.009\n",
      "INFO:tensorflow:loss = 0.07107046, step = 29301 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.1\n",
      "INFO:tensorflow:loss = 0.07304658, step = 29401 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.496\n",
      "INFO:tensorflow:loss = 0.025580278, step = 29501 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.928\n",
      "INFO:tensorflow:loss = 0.0109706605, step = 29601 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.236\n",
      "INFO:tensorflow:loss = 0.050516702, step = 29701 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.516\n",
      "INFO:tensorflow:loss = 0.027441502, step = 29801 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.315\n",
      "INFO:tensorflow:loss = 0.00014829269, step = 29901 (0.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.925\n",
      "INFO:tensorflow:loss = 0.0048629013, step = 30001 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.398\n",
      "INFO:tensorflow:loss = 0.10246532, step = 30101 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.656\n",
      "INFO:tensorflow:loss = 0.006725667, step = 30201 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.409\n",
      "INFO:tensorflow:loss = 0.027600303, step = 30301 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.33\n",
      "INFO:tensorflow:loss = 0.020341605, step = 30401 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.657\n",
      "INFO:tensorflow:loss = 0.03669761, step = 30501 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.814\n",
      "INFO:tensorflow:loss = 0.016639413, step = 30601 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.813\n",
      "INFO:tensorflow:loss = 0.036807742, step = 30701 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.71\n",
      "INFO:tensorflow:loss = 0.016331756, step = 30801 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.972\n",
      "INFO:tensorflow:loss = 0.0152390655, step = 30901 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.203\n",
      "INFO:tensorflow:loss = 0.0854864, step = 31001 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.389\n",
      "INFO:tensorflow:loss = 0.008512823, step = 31101 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.941\n",
      "INFO:tensorflow:loss = 0.01842174, step = 31201 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.692\n",
      "INFO:tensorflow:loss = 0.04167877, step = 31301 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.744\n",
      "INFO:tensorflow:loss = 0.005681064, step = 31401 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.523\n",
      "INFO:tensorflow:loss = 0.05001551, step = 31501 (0.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.238\n",
      "INFO:tensorflow:loss = 0.05346697, step = 31601 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.255\n",
      "INFO:tensorflow:loss = 0.0036199915, step = 31701 (0.320 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 316.604\n",
      "INFO:tensorflow:loss = 0.033709206, step = 31801 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.338\n",
      "INFO:tensorflow:loss = 0.007022989, step = 31901 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.137\n",
      "INFO:tensorflow:loss = 0.04764556, step = 32001 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.599\n",
      "INFO:tensorflow:loss = 0.013485136, step = 32101 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.823\n",
      "INFO:tensorflow:loss = 0.008174633, step = 32201 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.776\n",
      "INFO:tensorflow:loss = 0.025245465, step = 32301 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.968\n",
      "INFO:tensorflow:loss = 0.009170952, step = 32401 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.037\n",
      "INFO:tensorflow:loss = 0.031049363, step = 32501 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.839\n",
      "INFO:tensorflow:loss = 0.03124824, step = 32601 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.587\n",
      "INFO:tensorflow:loss = 0.016158927, step = 32701 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.877\n",
      "INFO:tensorflow:loss = 0.0068441285, step = 32801 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.5\n",
      "INFO:tensorflow:loss = 0.0073055187, step = 32901 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.053\n",
      "INFO:tensorflow:loss = 0.004400433, step = 33001 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.174\n",
      "INFO:tensorflow:loss = 0.060860608, step = 33101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.668\n",
      "INFO:tensorflow:loss = 0.08856924, step = 33201 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 321.871\n",
      "INFO:tensorflow:loss = 0.009618117, step = 33301 (0.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.361\n",
      "INFO:tensorflow:loss = 0.051448703, step = 33401 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.239\n",
      "INFO:tensorflow:loss = 0.003428059, step = 33501 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.835\n",
      "INFO:tensorflow:loss = 0.07767935, step = 33601 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.084\n",
      "INFO:tensorflow:loss = 0.018940076, step = 33701 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.682\n",
      "INFO:tensorflow:loss = 0.010316588, step = 33801 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.266\n",
      "INFO:tensorflow:loss = 0.027520237, step = 33901 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.24\n",
      "INFO:tensorflow:loss = 0.007522299, step = 34001 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.964\n",
      "INFO:tensorflow:loss = 0.069495276, step = 34101 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.648\n",
      "INFO:tensorflow:loss = 0.017848173, step = 34201 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.274\n",
      "INFO:tensorflow:loss = 0.04050123, step = 34301 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.041\n",
      "INFO:tensorflow:loss = 0.020775598, step = 34401 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.206\n",
      "INFO:tensorflow:loss = 0.08914474, step = 34501 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.284\n",
      "INFO:tensorflow:loss = 0.0383292, step = 34601 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.852\n",
      "INFO:tensorflow:loss = 0.018478192, step = 34701 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.096\n",
      "INFO:tensorflow:loss = 0.060178842, step = 34801 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.841\n",
      "INFO:tensorflow:loss = 0.0036822173, step = 34901 (0.321 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.241\n",
      "INFO:tensorflow:loss = 0.0118706, step = 35001 (0.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.415\n",
      "INFO:tensorflow:loss = 0.06938365, step = 35101 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.803\n",
      "INFO:tensorflow:loss = 0.0055069462, step = 35201 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.47\n",
      "INFO:tensorflow:loss = 0.014765613, step = 35301 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.641\n",
      "INFO:tensorflow:loss = 0.04678248, step = 35401 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 319.03\n",
      "INFO:tensorflow:loss = 0.0483696, step = 35501 (0.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.783\n",
      "INFO:tensorflow:loss = 0.044839565, step = 35601 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.709\n",
      "INFO:tensorflow:loss = 0.08575373, step = 35701 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.435\n",
      "INFO:tensorflow:loss = 0.017645124, step = 35801 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.754\n",
      "INFO:tensorflow:loss = 0.017543335, step = 35901 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.785\n",
      "INFO:tensorflow:loss = 0.005136019, step = 36001 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.31\n",
      "INFO:tensorflow:loss = 0.042684756, step = 36101 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.961\n",
      "INFO:tensorflow:loss = 0.011314306, step = 36201 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.401\n",
      "INFO:tensorflow:loss = 0.025708392, step = 36301 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.932\n",
      "INFO:tensorflow:loss = 0.008726337, step = 36401 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.994\n",
      "INFO:tensorflow:loss = 0.0067686657, step = 36501 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.096\n",
      "INFO:tensorflow:loss = 0.016456544, step = 36601 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.063\n",
      "INFO:tensorflow:loss = 0.006296455, step = 36701 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.069\n",
      "INFO:tensorflow:loss = 0.031662535, step = 36801 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.398\n",
      "INFO:tensorflow:loss = 0.014697939, step = 36901 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.404\n",
      "INFO:tensorflow:loss = 0.007511735, step = 37001 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 310.058\n",
      "INFO:tensorflow:loss = 0.0058631073, step = 37101 (0.322 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.47\n",
      "INFO:tensorflow:loss = 0.0010046071, step = 37201 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.541\n",
      "INFO:tensorflow:loss = 0.013231819, step = 37301 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.546\n",
      "INFO:tensorflow:loss = 0.028619831, step = 37401 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.598\n",
      "INFO:tensorflow:loss = 0.012039711, step = 37501 (0.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.843\n",
      "INFO:tensorflow:loss = 0.001051567, step = 37601 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.041\n",
      "INFO:tensorflow:loss = 0.056832343, step = 37701 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 313.927\n",
      "INFO:tensorflow:loss = 0.033166397, step = 37801 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.303\n",
      "INFO:tensorflow:loss = 0.00035487156, step = 37901 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.673\n",
      "INFO:tensorflow:loss = 0.04118822, step = 38001 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.241\n",
      "INFO:tensorflow:loss = 0.023454571, step = 38101 (0.335 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.79\n",
      "INFO:tensorflow:loss = 0.00840847, step = 38201 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.158\n",
      "INFO:tensorflow:loss = 0.00660785, step = 38301 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.297\n",
      "INFO:tensorflow:loss = 0.011355675, step = 38401 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 297.845\n",
      "INFO:tensorflow:loss = 0.013734322, step = 38501 (0.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 260.557\n",
      "INFO:tensorflow:loss = 0.029144779, step = 38601 (0.386 sec)\n",
      "INFO:tensorflow:global_step/sec: 267.322\n",
      "INFO:tensorflow:loss = 0.02522317, step = 38701 (0.372 sec)\n",
      "INFO:tensorflow:global_step/sec: 268.534\n",
      "INFO:tensorflow:loss = 0.012233912, step = 38801 (0.372 sec)\n",
      "INFO:tensorflow:global_step/sec: 309.218\n",
      "INFO:tensorflow:loss = 0.017001547, step = 38901 (0.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 298.385\n",
      "INFO:tensorflow:loss = 0.02655631, step = 39001 (0.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.109\n",
      "INFO:tensorflow:loss = 0.0017474877, step = 39101 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.841\n",
      "INFO:tensorflow:loss = 0.038495496, step = 39201 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.004\n",
      "INFO:tensorflow:loss = 0.03635207, step = 39301 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 312.734\n",
      "INFO:tensorflow:loss = 0.031774502, step = 39401 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.981\n",
      "INFO:tensorflow:loss = 0.012629777, step = 39501 (0.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.042\n",
      "INFO:tensorflow:loss = 0.0143067865, step = 39601 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.064\n",
      "INFO:tensorflow:loss = 0.006867146, step = 39701 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.018\n",
      "INFO:tensorflow:loss = 0.02735231, step = 39801 (0.326 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 318.566\n",
      "INFO:tensorflow:loss = 0.04105402, step = 39901 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.13\n",
      "INFO:tensorflow:loss = 0.0055373367, step = 40001 (0.329 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.152\n",
      "INFO:tensorflow:loss = 0.04081005, step = 40101 (0.319 sec)\n",
      "INFO:tensorflow:global_step/sec: 316.413\n",
      "INFO:tensorflow:loss = 0.0030397002, step = 40201 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 308.276\n",
      "INFO:tensorflow:loss = 0.009227021, step = 40301 (0.324 sec)\n",
      "INFO:tensorflow:global_step/sec: 318.187\n",
      "INFO:tensorflow:loss = 0.020871505, step = 40401 (0.314 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.454\n",
      "INFO:tensorflow:loss = 0.0074155447, step = 40501 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.614\n",
      "INFO:tensorflow:loss = 0.012822457, step = 40601 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 314.096\n",
      "INFO:tensorflow:loss = 0.0347273, step = 40701 (0.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.237\n",
      "INFO:tensorflow:loss = 0.018538067, step = 40801 (0.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 317.323\n",
      "INFO:tensorflow:loss = 0.010809799, step = 40901 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.565\n",
      "INFO:tensorflow:loss = 0.0121064, step = 41001 (0.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.642\n",
      "INFO:tensorflow:loss = 0.018093586, step = 41101 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.616\n",
      "INFO:tensorflow:loss = 0.028048482, step = 41201 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 304.176\n",
      "INFO:tensorflow:loss = 0.01686172, step = 41301 (0.329 sec)\n",
      "INFO:tensorflow:global_step/sec: 322.106\n",
      "INFO:tensorflow:loss = 0.002214645, step = 41401 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 306.092\n",
      "INFO:tensorflow:loss = 0.006440325, step = 41501 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 299.628\n",
      "INFO:tensorflow:loss = 0.008892274, step = 41601 (0.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.896\n",
      "INFO:tensorflow:loss = 0.013109217, step = 41701 (0.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 274.02\n",
      "INFO:tensorflow:loss = 0.05503158, step = 41801 (0.365 sec)\n",
      "INFO:tensorflow:global_step/sec: 287.41\n",
      "INFO:tensorflow:loss = 0.05566992, step = 41901 (0.348 sec)\n",
      "INFO:tensorflow:global_step/sec: 305.496\n",
      "INFO:tensorflow:loss = 0.0037532556, step = 42001 (0.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.012\n",
      "INFO:tensorflow:loss = 0.0033683684, step = 42101 (0.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 311.946\n",
      "INFO:tensorflow:loss = 0.03062315, step = 42201 (0.320 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.293\n",
      "INFO:tensorflow:loss = 0.01057074, step = 42301 (0.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 299.094\n",
      "INFO:tensorflow:loss = 0.019507855, step = 42401 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 242.617\n",
      "INFO:tensorflow:loss = 0.016443389, step = 42501 (0.412 sec)\n",
      "INFO:tensorflow:global_step/sec: 296.454\n",
      "INFO:tensorflow:loss = 0.027688995, step = 42601 (0.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 291.786\n",
      "INFO:tensorflow:loss = 0.015084272, step = 42701 (0.342 sec)\n",
      "INFO:tensorflow:global_step/sec: 299.833\n",
      "INFO:tensorflow:loss = 0.017545052, step = 42801 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 303.33\n",
      "INFO:tensorflow:loss = 0.014100645, step = 42901 (0.330 sec)\n",
      "INFO:tensorflow:global_step/sec: 233.927\n",
      "INFO:tensorflow:loss = 0.0036637802, step = 43001 (0.427 sec)\n",
      "INFO:tensorflow:global_step/sec: 226.526\n",
      "INFO:tensorflow:loss = 0.0048309015, step = 43101 (0.442 sec)\n",
      "INFO:tensorflow:global_step/sec: 301.572\n",
      "INFO:tensorflow:loss = 0.018219372, step = 43201 (0.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 300.219\n",
      "INFO:tensorflow:loss = 0.040053643, step = 43301 (0.333 sec)\n",
      "INFO:tensorflow:global_step/sec: 293.992\n",
      "INFO:tensorflow:loss = 0.008673076, step = 43401 (0.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 249.407\n",
      "INFO:tensorflow:loss = 0.013726322, step = 43501 (0.401 sec)\n",
      "INFO:tensorflow:global_step/sec: 288.421\n",
      "INFO:tensorflow:loss = 0.09631184, step = 43601 (0.347 sec)\n",
      "INFO:tensorflow:global_step/sec: 315.647\n",
      "INFO:tensorflow:loss = 0.051468477, step = 43701 (0.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 297.846\n",
      "INFO:tensorflow:loss = 0.043439846, step = 43801 (0.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 307.393\n",
      "INFO:tensorflow:loss = 0.02744491, step = 43901 (0.326 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpef5o9cas/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.012941079.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.dnn.DNNClassifier at 0x7fc43ead2cf8>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [tf.feature_column.numeric_column(\"X\", shape=[28 * 28])] # 定义特征列\n",
    "dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                     feature_columns=feature_cols) #实例化DNN分类器;hidden_units:两个隐藏层，分别300和100个神经元;n_classes:分10类，即输出层为10个神经元;feature_columns:特征列\n",
    "\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True) # 定义输入函数;x,y:实例与标签;num_epoch:数据迭代周期;batch_size: 类似小批量梯度下降中每批量的大小。shuffle：为True时将队列打乱。\n",
    "dnn_clf.train(input_fn=input_fn) # 开始训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-12-10-06:30:27\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpef5o9cas/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-12-10-06:30:27\n",
      "INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9803, average_loss = 0.10239366, global_step = 44000, loss = 12.961223\n"
     ]
    }
   ],
   "source": [
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_test}, y=y_test, shuffle=False) # 测试集的输入函数\n",
    "eval_results = dnn_clf.evaluate(input_fn=test_input_fn) # 对训练好的DNN分类器输入测试集来进行评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9803,\n",
       " 'average_loss': 0.10239366,\n",
       " 'global_step': 44000,\n",
       " 'loss': 12.961223}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_results # 评估结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpef5o9cas/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'class_ids': array([7]),\n",
       " 'classes': array([b'7'], dtype=object),\n",
       " 'logits': array([ -7.381111 ,   0.7214664,  -0.6627069,   4.4830103,  -1.9302189,\n",
       "         -6.9162908, -21.178396 ,  22.122177 ,  -1.8814406,   3.0483098],\n",
       "       dtype=float32),\n",
       " 'probabilities': array([1.5377463e-13, 5.0791293e-10, 1.2724796e-10, 2.1847752e-08,\n",
       "        3.5824226e-11, 2.4476756e-13, 1.5660265e-19, 1.0000000e+00,\n",
       "        3.7614998e-11, 5.2038462e-09], dtype=float32)}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_iter = dnn_clf.predict(input_fn=test_input_fn) # 输入测试集进行预测,返回生成器\n",
    "y_pred = list(y_pred_iter) # 生成器转换为列表\n",
    "y_pred[0] # 第一个预测"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Using plain TensorFlow\n",
    "## 使用素的Tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST 输入\n",
    "n_hidden1 = 300 # 300个神经元的隐藏层\n",
    "n_hidden2 = 100 # 100个神经元的隐藏层\n",
    "n_outputs = 10 # 10个输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph() # In[1]定义的函数\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\") # 定义占位符，类型为float32，大小为输入层大小;占位符相当于形参，先定义结构，后续再进行赋值。\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Val accuracy: 0.9146\n",
      "1 Batch accuracy: 0.92 Val accuracy: 0.936\n",
      "2 Batch accuracy: 0.96 Val accuracy: 0.945\n",
      "3 Batch accuracy: 0.92 Val accuracy: 0.9512\n",
      "4 Batch accuracy: 0.98 Val accuracy: 0.9558\n",
      "5 Batch accuracy: 0.96 Val accuracy: 0.9566\n",
      "6 Batch accuracy: 1.0 Val accuracy: 0.9612\n",
      "7 Batch accuracy: 0.94 Val accuracy: 0.963\n",
      "8 Batch accuracy: 0.98 Val accuracy: 0.9652\n",
      "9 Batch accuracy: 0.96 Val accuracy: 0.966\n",
      "10 Batch accuracy: 0.92 Val accuracy: 0.9688\n",
      "11 Batch accuracy: 0.98 Val accuracy: 0.969\n",
      "12 Batch accuracy: 0.98 Val accuracy: 0.967\n",
      "13 Batch accuracy: 0.98 Val accuracy: 0.9706\n",
      "14 Batch accuracy: 1.0 Val accuracy: 0.9714\n",
      "15 Batch accuracy: 0.94 Val accuracy: 0.9732\n",
      "16 Batch accuracy: 1.0 Val accuracy: 0.9736\n",
      "17 Batch accuracy: 1.0 Val accuracy: 0.9742\n",
      "18 Batch accuracy: 1.0 Val accuracy: 0.9746\n",
      "19 Batch accuracy: 0.98 Val accuracy: 0.9748\n",
      "20 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "21 Batch accuracy: 1.0 Val accuracy: 0.9752\n",
      "22 Batch accuracy: 0.98 Val accuracy: 0.9764\n",
      "23 Batch accuracy: 0.98 Val accuracy: 0.9752\n",
      "24 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "25 Batch accuracy: 1.0 Val accuracy: 0.977\n",
      "26 Batch accuracy: 0.98 Val accuracy: 0.9778\n",
      "27 Batch accuracy: 1.0 Val accuracy: 0.9774\n",
      "28 Batch accuracy: 0.96 Val accuracy: 0.9754\n",
      "29 Batch accuracy: 0.98 Val accuracy: 0.9776\n",
      "30 Batch accuracy: 1.0 Val accuracy: 0.9756\n",
      "31 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "32 Batch accuracy: 0.98 Val accuracy: 0.9772\n",
      "33 Batch accuracy: 0.98 Val accuracy: 0.979\n",
      "34 Batch accuracy: 1.0 Val accuracy: 0.9784\n",
      "35 Batch accuracy: 1.0 Val accuracy: 0.9778\n",
      "36 Batch accuracy: 0.98 Val accuracy: 0.978\n",
      "37 Batch accuracy: 1.0 Val accuracy: 0.9776\n",
      "38 Batch accuracy: 1.0 Val accuracy: 0.9792\n",
      "39 Batch accuracy: 1.0 Val accuracy: 0.9776\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Val accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # or better, use save_path\n",
    "    X_new_scaled = X_test[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted classes: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n",
      "Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"Predicted classes:\", y_pred)\n",
    "print(\"Actual classes:   \", y_test[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using `dense()` instead of `neuron_layer()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: previous releases of the book used `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* a few more differences are presented in chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "    y_proba = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.9 Validation accuracy: 0.9024\n",
      "1 Batch accuracy: 0.92 Validation accuracy: 0.9254\n",
      "2 Batch accuracy: 0.94 Validation accuracy: 0.9372\n",
      "3 Batch accuracy: 0.9 Validation accuracy: 0.9416\n",
      "4 Batch accuracy: 0.94 Validation accuracy: 0.9472\n",
      "5 Batch accuracy: 0.94 Validation accuracy: 0.9512\n",
      "6 Batch accuracy: 1.0 Validation accuracy: 0.9548\n",
      "7 Batch accuracy: 0.94 Validation accuracy: 0.961\n",
      "8 Batch accuracy: 0.96 Validation accuracy: 0.962\n",
      "9 Batch accuracy: 0.94 Validation accuracy: 0.9648\n",
      "10 Batch accuracy: 0.92 Validation accuracy: 0.9656\n",
      "11 Batch accuracy: 0.98 Validation accuracy: 0.9668\n",
      "12 Batch accuracy: 0.98 Validation accuracy: 0.9684\n",
      "13 Batch accuracy: 0.98 Validation accuracy: 0.9702\n",
      "14 Batch accuracy: 1.0 Validation accuracy: 0.9696\n",
      "15 Batch accuracy: 0.94 Validation accuracy: 0.9718\n",
      "16 Batch accuracy: 0.98 Validation accuracy: 0.9728\n",
      "17 Batch accuracy: 1.0 Validation accuracy: 0.973\n",
      "18 Batch accuracy: 0.98 Validation accuracy: 0.9748\n",
      "19 Batch accuracy: 0.98 Validation accuracy: 0.9756\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"Batch accuracy:\", acc_batch, \"Validation accuracy:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define the directory to write the TensorBoard logs to:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hey! Why don't we implement early stopping? For this, we are going to need to use the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 \tValidation accuracy: 92.180% \tLoss: 0.30208\n",
      "Epoch: 5 \tValidation accuracy: 95.980% \tLoss: 0.15037\n",
      "Epoch: 10 \tValidation accuracy: 97.100% \tLoss: 0.11160\n",
      "Epoch: 15 \tValidation accuracy: 97.700% \tLoss: 0.09562\n",
      "Epoch: 20 \tValidation accuracy: 97.840% \tLoss: 0.08309\n",
      "Epoch: 25 \tValidation accuracy: 98.040% \tLoss: 0.07706\n",
      "Epoch: 30 \tValidation accuracy: 98.140% \tLoss: 0.07287\n",
      "Epoch: 35 \tValidation accuracy: 98.280% \tLoss: 0.07133\n",
      "Epoch: 40 \tValidation accuracy: 98.220% \tLoss: 0.06968\n",
      "Epoch: 45 \tValidation accuracy: 98.220% \tLoss: 0.06993\n",
      "Epoch: 50 \tValidation accuracy: 98.160% \tLoss: 0.07093\n",
      "Epoch: 55 \tValidation accuracy: 98.280% \tLoss: 0.06994\n",
      "Epoch: 60 \tValidation accuracy: 98.200% \tLoss: 0.06894\n",
      "Epoch: 65 \tValidation accuracy: 98.260% \tLoss: 0.06906\n",
      "Epoch: 70 \tValidation accuracy: 98.220% \tLoss: 0.07057\n",
      "Epoch: 75 \tValidation accuracy: 98.280% \tLoss: 0.06963\n",
      "Epoch: 80 \tValidation accuracy: 98.320% \tLoss: 0.07264\n",
      "Epoch: 85 \tValidation accuracy: 98.200% \tLoss: 0.07403\n",
      "Epoch: 90 \tValidation accuracy: 98.300% \tLoss: 0.07332\n",
      "Epoch: 95 \tValidation accuracy: 98.180% \tLoss: 0.07535\n",
      "Epoch: 100 \tValidation accuracy: 98.260% \tLoss: 0.07542\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # if the checkpoint file exists, restore the model and load the epoch number\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"Training was interrupted. Continuing at epoch\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"Epoch:\", epoch,\n",
    "                  \"\\tValidation accuracy: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\tLoss: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9796"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
